mirror of
https://github.com/Tautulli/Tautulli.git
synced 2025-08-19 21:03:21 -07:00
Update ipwhois to 1.1.0
This commit is contained in:
JonnyWong16
parent
4d6279a626
commit
84ce4758d1
13 changed files with 5041 additions and 649 deletions
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright (c) 2013, 2014, 2015, 2016 Philip Hane
|
||||
# Copyright (c) 2013-2019 Philip Hane
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -26,46 +26,44 @@ import sys
|
|||
import socket
|
||||
import dns.resolver
|
||||
import json
|
||||
from collections import namedtuple
|
||||
import logging
|
||||
from time import sleep
|
||||
|
||||
# Import the dnspython3 rdtypes to fix the dynamic import problem when frozen.
|
||||
# Import the dnspython rdtypes to fix the dynamic import problem when frozen.
|
||||
import dns.rdtypes.ANY.TXT # @UnusedImport
|
||||
|
||||
from .exceptions import (IPDefinedError, ASNRegistryError, ASNLookupError,
|
||||
BlacklistError, WhoisLookupError, HTTPLookupError,
|
||||
HostLookupError, HTTPRateLimitError)
|
||||
from .exceptions import (IPDefinedError, ASNLookupError, BlacklistError,
|
||||
WhoisLookupError, HTTPLookupError, HostLookupError,
|
||||
HTTPRateLimitError, WhoisRateLimitError)
|
||||
from .whois import RIR_WHOIS
|
||||
from .asn import ASN_ORIGIN_WHOIS
|
||||
from .utils import ipv4_is_defined, ipv6_is_defined
|
||||
|
||||
if sys.version_info >= (3, 3): # pragma: no cover
|
||||
from ipaddress import (ip_address,
|
||||
IPv4Address,
|
||||
IPv6Address,
|
||||
ip_network,
|
||||
summarize_address_range,
|
||||
collapse_addresses)
|
||||
IPv6Address)
|
||||
else: # pragma: no cover
|
||||
from ipaddr import (IPAddress as ip_address,
|
||||
IPv4Address,
|
||||
IPv6Address,
|
||||
IPNetwork as ip_network,
|
||||
summarize_address_range,
|
||||
collapse_address_list as collapse_addresses)
|
||||
IPv6Address)
|
||||
|
||||
try: # pragma: no cover
|
||||
from urllib.request import (OpenerDirector,
|
||||
ProxyHandler,
|
||||
build_opener,
|
||||
Request,
|
||||
URLError)
|
||||
URLError,
|
||||
HTTPError)
|
||||
from urllib.parse import urlencode
|
||||
except ImportError: # pragma: no cover
|
||||
from urllib2 import (OpenerDirector,
|
||||
ProxyHandler,
|
||||
build_opener,
|
||||
Request,
|
||||
URLError)
|
||||
URLError,
|
||||
HTTPError)
|
||||
from urllib import urlencode
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -73,22 +71,6 @@ log = logging.getLogger(__name__)
|
|||
# POSSIBLY UPDATE TO USE RDAP
|
||||
ARIN = 'http://whois.arin.net/rest/nets;q={0}?showDetails=true&showARIN=true'
|
||||
|
||||
# National Internet Registry
|
||||
NIR = {
|
||||
'jpnic': {
|
||||
'url': ('http://whois.nic.ad.jp/cgi-bin/whois_gw?lang=%2Fe&key={0}'
|
||||
'&submit=query'),
|
||||
'request_type': 'GET',
|
||||
'request_headers': {'Accept': 'text/html'}
|
||||
},
|
||||
'krnic': {
|
||||
'url': 'http://whois.kisa.or.kr/eng/whois.jsc',
|
||||
'request_type': 'POST',
|
||||
'request_headers': {'Accept': 'text/html'},
|
||||
'form_data_ip_field': 'query'
|
||||
}
|
||||
}
|
||||
|
||||
CYMRU_WHOIS = 'whois.cymru.com'
|
||||
|
||||
IPV4_DNS_ZONE = '{0}.origin.asn.cymru.com'
|
||||
|
|
@ -115,12 +97,15 @@ class Net:
|
|||
The class for performing network queries.
|
||||
|
||||
Args:
|
||||
address: An IPv4 or IPv6 address in string format.
|
||||
timeout: The default timeout for socket connections in seconds.
|
||||
proxy_opener: The urllib.request.OpenerDirector request for proxy
|
||||
support or None.
|
||||
allow_permutations: Use additional methods if DNS lookups to Cymru
|
||||
fail.
|
||||
address (:obj:`str`/:obj:`int`/:obj:`IPv4Address`/:obj:`IPv6Address`):
|
||||
An IPv4 or IPv6 address
|
||||
timeout (:obj:`int`): The default timeout for socket connections in
|
||||
seconds. Defaults to 5.
|
||||
proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for
|
||||
proxy support. Defaults to None.
|
||||
allow_permutations (:obj:`bool`): Allow net.Net() to use additional
|
||||
methods if DNS lookups to Cymru fail. *WARNING* deprecated in
|
||||
favor of new argument asn_methods. Defaults to False.
|
||||
|
||||
Raises:
|
||||
IPDefinedError: The address provided is defined (does not need to be
|
||||
|
|
@ -128,7 +113,7 @@ class Net:
|
|||
"""
|
||||
|
||||
def __init__(self, address, timeout=5, proxy_opener=None,
|
||||
allow_permutations=True):
|
||||
allow_permutations=False):
|
||||
|
||||
# IPv4Address or IPv6Address
|
||||
if isinstance(address, IPv4Address) or isinstance(
|
||||
|
|
@ -147,6 +132,13 @@ class Net:
|
|||
# Allow other than DNS lookups for ASNs.
|
||||
self.allow_permutations = allow_permutations
|
||||
|
||||
if self.allow_permutations:
|
||||
|
||||
from warnings import warn
|
||||
warn('allow_permutations has been deprecated and will be removed. '
|
||||
'It is no longer needed, due to the deprecation of asn_alts, '
|
||||
'and the addition of the asn_methods argument.')
|
||||
|
||||
self.dns_resolver = dns.resolver.Resolver()
|
||||
self.dns_resolver.timeout = timeout
|
||||
self.dns_resolver.lifetime = timeout
|
||||
|
|
@ -227,59 +219,38 @@ class Net:
|
|||
|
||||
self.dns_zone = IPV6_DNS_ZONE.format(self.reversed)
|
||||
|
||||
def get_asn_dns(self, result=None):
|
||||
def lookup_asn(self, *args, **kwargs):
|
||||
"""
|
||||
Temporary wrapper for IP ASN lookups (moved to
|
||||
asn.IPASN.lookup()). This will be removed in a future
|
||||
release.
|
||||
"""
|
||||
|
||||
from warnings import warn
|
||||
warn('Net.lookup_asn() has been deprecated and will be removed. '
|
||||
'You should now use asn.IPASN.lookup() for IP ASN lookups.')
|
||||
from .asn import IPASN
|
||||
response = None
|
||||
ipasn = IPASN(self)
|
||||
return ipasn.lookup(*args, **kwargs), response
|
||||
|
||||
def get_asn_dns(self):
|
||||
"""
|
||||
The function for retrieving ASN information for an IP address from
|
||||
Cymru via port 53 (DNS).
|
||||
|
||||
Args:
|
||||
result: Optional result object. This bypasses the ASN lookup.
|
||||
|
||||
Returns:
|
||||
Dictionary: A dictionary containing the following keys:
|
||||
asn (String) - The Autonomous System Number.
|
||||
asn_date (String) - The ASN Allocation date.
|
||||
asn_registry (String) - The assigned ASN registry.
|
||||
asn_cidr (String) - The assigned ASN CIDR.
|
||||
asn_country_code (String) - The assigned ASN country code.
|
||||
list: The raw ASN data.
|
||||
|
||||
Raises:
|
||||
ASNRegistryError: The ASN registry is not known.
|
||||
ASNLookupError: The ASN lookup failed.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
if result is None:
|
||||
|
||||
log.debug('ASN query for {0}'.format(self.dns_zone))
|
||||
data = self.dns_resolver.query(self.dns_zone, 'TXT')
|
||||
temp = str(data[0]).split('|')
|
||||
|
||||
else:
|
||||
|
||||
temp = result
|
||||
|
||||
# Parse out the ASN information.
|
||||
ret = {'asn_registry': temp[3].strip(' \n')}
|
||||
|
||||
if ret['asn_registry'] not in RIR_WHOIS.keys():
|
||||
|
||||
raise ASNRegistryError(
|
||||
'ASN registry {0} is not known.'.format(
|
||||
ret['asn_registry'])
|
||||
)
|
||||
|
||||
ret['asn'] = temp[0].strip(' "\n')
|
||||
ret['asn_cidr'] = temp[1].strip(' \n')
|
||||
ret['asn_country_code'] = temp[2].strip(' \n').upper()
|
||||
ret['asn_date'] = temp[4].strip(' "\n')
|
||||
|
||||
return ret
|
||||
|
||||
except ASNRegistryError:
|
||||
|
||||
raise
|
||||
log.debug('ASN query for {0}'.format(self.dns_zone))
|
||||
data = self.dns_resolver.query(self.dns_zone, 'TXT')
|
||||
return list(data)
|
||||
|
||||
except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers,
|
||||
dns.resolver.NoAnswer, dns.exception.Timeout) as e:
|
||||
|
|
@ -289,85 +260,98 @@ class Net:
|
|||
e.__class__.__name__, self.address_str)
|
||||
)
|
||||
|
||||
except:
|
||||
except: # pragma: no cover
|
||||
|
||||
raise ASNLookupError(
|
||||
'ASN lookup failed for {0}.'.format(self.address_str)
|
||||
)
|
||||
|
||||
def get_asn_whois(self, retry_count=3, result=None):
|
||||
def get_asn_verbose_dns(self, asn=None):
|
||||
"""
|
||||
The function for retrieving the information for an ASN from
|
||||
Cymru via port 53 (DNS). This is needed since IP to ASN mapping via
|
||||
Cymru DNS does not return the ASN Description like Cymru Whois does.
|
||||
|
||||
Args:
|
||||
asn (:obj:`str`): The AS number (required).
|
||||
|
||||
Returns:
|
||||
str: The raw ASN data.
|
||||
|
||||
Raises:
|
||||
ASNLookupError: The ASN lookup failed.
|
||||
"""
|
||||
|
||||
if asn[0:2] != 'AS':
|
||||
|
||||
asn = 'AS{0}'.format(asn)
|
||||
|
||||
zone = '{0}.asn.cymru.com'.format(asn)
|
||||
|
||||
try:
|
||||
|
||||
log.debug('ASN verbose query for {0}'.format(zone))
|
||||
data = self.dns_resolver.query(zone, 'TXT')
|
||||
return str(data[0])
|
||||
|
||||
except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers,
|
||||
dns.resolver.NoAnswer, dns.exception.Timeout) as e:
|
||||
|
||||
raise ASNLookupError(
|
||||
'ASN lookup failed (DNS {0}) for {1}.'.format(
|
||||
e.__class__.__name__, asn)
|
||||
)
|
||||
|
||||
except: # pragma: no cover
|
||||
|
||||
raise ASNLookupError(
|
||||
'ASN lookup failed for {0}.'.format(asn)
|
||||
)
|
||||
|
||||
def get_asn_whois(self, retry_count=3):
|
||||
"""
|
||||
The function for retrieving ASN information for an IP address from
|
||||
Cymru via port 43/tcp (WHOIS).
|
||||
|
||||
Args:
|
||||
retry_count: The number of times to retry in case socket errors,
|
||||
timeouts, connection resets, etc. are encountered.
|
||||
result: Optional result object. This bypasses the ASN lookup.
|
||||
retry_count (:obj:`int`): The number of times to retry in case
|
||||
socket errors, timeouts, connection resets, etc. are
|
||||
encountered. Defaults to 3.
|
||||
|
||||
Returns:
|
||||
Dictionary: A dictionary containing the following keys:
|
||||
asn (String) - The Autonomous System Number.
|
||||
asn_date (String) - The ASN Allocation date.
|
||||
asn_registry (String) - The assigned ASN registry.
|
||||
asn_cidr (String) - The assigned ASN CIDR.
|
||||
asn_country_code (String) - The assigned ASN country code.
|
||||
str: The raw ASN data.
|
||||
|
||||
Raises:
|
||||
ASNRegistryError: The ASN registry is not known.
|
||||
ASNLookupError: The ASN lookup failed.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
if result is None:
|
||||
# Create the connection for the Cymru whois query.
|
||||
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
conn.settimeout(self.timeout)
|
||||
log.debug('ASN query for {0}'.format(self.address_str))
|
||||
conn.connect((CYMRU_WHOIS, 43))
|
||||
|
||||
# Create the connection for the Cymru whois query.
|
||||
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
conn.settimeout(self.timeout)
|
||||
log.debug('ASN query for {0}'.format(self.address_str))
|
||||
conn.connect((CYMRU_WHOIS, 43))
|
||||
# Query the Cymru whois server, and store the results.
|
||||
conn.send((
|
||||
' -r -a -c -p -f {0}{1}'.format(
|
||||
self.address_str, '\r\n')
|
||||
).encode())
|
||||
|
||||
# Query the Cymru whois server, and store the results.
|
||||
conn.send((
|
||||
' -r -a -c -p -f -o {0}{1}'.format(
|
||||
self.address_str, '\r\n')
|
||||
).encode())
|
||||
data = ''
|
||||
while True:
|
||||
|
||||
data = ''
|
||||
while True:
|
||||
d = conn.recv(4096).decode()
|
||||
data += d
|
||||
|
||||
d = conn.recv(4096).decode()
|
||||
data += d
|
||||
if not d:
|
||||
|
||||
if not d:
|
||||
break
|
||||
|
||||
break
|
||||
conn.close()
|
||||
|
||||
conn.close()
|
||||
|
||||
else:
|
||||
|
||||
data = result
|
||||
|
||||
# Parse out the ASN information.
|
||||
temp = str(data).split('|')
|
||||
|
||||
ret = {'asn_registry': temp[4].strip(' \n')}
|
||||
|
||||
if ret['asn_registry'] not in RIR_WHOIS.keys():
|
||||
|
||||
raise ASNRegistryError(
|
||||
'ASN registry {0} is not known.'.format(
|
||||
ret['asn_registry'])
|
||||
)
|
||||
|
||||
ret['asn'] = temp[0].strip(' \n')
|
||||
ret['asn_cidr'] = temp[2].strip(' \n')
|
||||
ret['asn_country_code'] = temp[3].strip(' \n').upper()
|
||||
ret['asn_date'] = temp[5].strip(' \n')
|
||||
|
||||
return ret
|
||||
return str(data)
|
||||
|
||||
except (socket.timeout, socket.error) as e: # pragma: no cover
|
||||
|
||||
|
|
@ -384,17 +368,13 @@ class Net:
|
|||
'ASN lookup failed for {0}.'.format(self.address_str)
|
||||
)
|
||||
|
||||
except ASNRegistryError:
|
||||
|
||||
raise
|
||||
|
||||
except:
|
||||
except: # pragma: no cover
|
||||
|
||||
raise ASNLookupError(
|
||||
'ASN lookup failed for {0}.'.format(self.address_str)
|
||||
)
|
||||
|
||||
def get_asn_http(self, retry_count=3, result=None, extra_org_map=None):
|
||||
def get_asn_http(self, retry_count=3):
|
||||
"""
|
||||
The function for retrieving ASN information for an IP address from
|
||||
Arin via port 80 (HTTP). Currently limited to fetching asn_registry
|
||||
|
|
@ -403,94 +383,29 @@ class Net:
|
|||
chance fallback call behind ASN DNS & ASN Whois lookups.
|
||||
|
||||
Args:
|
||||
retry_count: The number of times to retry in case socket errors,
|
||||
timeouts, connection resets, etc. are encountered.
|
||||
result: Optional result object. This bypasses the ASN lookup.
|
||||
extra_org_map: Dictionary mapping org handles to RIRs. This is for
|
||||
limited cases where ARIN REST (ASN fallback HTTP lookup) does
|
||||
not show an RIR as the org handle e.g., DNIC (which is now the
|
||||
built in ORG_MAP) e.g., {'DNIC': 'arin'}. Valid RIR values are
|
||||
(note the case-sensitive - this is meant to match the REST
|
||||
result): 'ARIN', 'RIPE', 'apnic', 'lacnic', 'afrinic'
|
||||
retry_count (:obj:`int`): The number of times to retry in case
|
||||
socket errors, timeouts, connection resets, etc. are
|
||||
encountered. Defaults to 3.
|
||||
|
||||
Returns:
|
||||
Dictionary: A dictionary containing the following keys:
|
||||
asn (String) - None, can't retrieve with this method.
|
||||
asn_date (String) - None, can't retrieve with this method.
|
||||
asn_registry (String) - The assigned ASN registry.
|
||||
asn_cidr (String) - None, can't retrieve with this method.
|
||||
asn_country_code (String) - None, can't retrieve with this
|
||||
method.
|
||||
dict: The ASN data in json format.
|
||||
|
||||
Raises:
|
||||
ASNRegistryError: The ASN registry is not known.
|
||||
ASNLookupError: The ASN lookup failed.
|
||||
"""
|
||||
|
||||
# Set the org_map. Map the orgRef handle to an RIR.
|
||||
org_map = ORG_MAP.copy()
|
||||
try:
|
||||
|
||||
org_map.update(extra_org_map)
|
||||
|
||||
except (TypeError, ValueError, IndexError, KeyError):
|
||||
|
||||
pass
|
||||
|
||||
try:
|
||||
|
||||
if result is None:
|
||||
|
||||
# Lets attempt to get the ASN registry information from
|
||||
# ARIN.
|
||||
log.debug('ASN query for {0}'.format(self.address_str))
|
||||
response = self.get_http_json(
|
||||
url=str(ARIN).format(self.address_str),
|
||||
retry_count=retry_count,
|
||||
headers={'Accept': 'application/json'}
|
||||
# Lets attempt to get the ASN registry information from
|
||||
# ARIN.
|
||||
log.debug('ASN query for {0}'.format(self.address_str))
|
||||
response = self.get_http_json(
|
||||
url=str(ARIN).format(self.address_str),
|
||||
retry_count=retry_count,
|
||||
headers={'Accept': 'application/json'}
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
response = result
|
||||
|
||||
asn_data = {
|
||||
'asn_registry': None,
|
||||
'asn': None,
|
||||
'asn_cidr': None,
|
||||
'asn_country_code': None,
|
||||
'asn_date': None
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
net_list = response['nets']['net']
|
||||
|
||||
if not isinstance(net_list, list):
|
||||
net_list = [net_list]
|
||||
|
||||
except (KeyError, TypeError):
|
||||
|
||||
log.debug('No networks found')
|
||||
net_list = []
|
||||
|
||||
for n in net_list:
|
||||
|
||||
try:
|
||||
|
||||
asn_data['asn_registry'] = (
|
||||
org_map[n['orgRef']['@handle'].upper()]
|
||||
)
|
||||
|
||||
except KeyError as e:
|
||||
|
||||
log.debug('Could not parse ASN registry via HTTP: '
|
||||
'{0}'.format(str(e)))
|
||||
raise ASNRegistryError('ASN registry lookup failed.')
|
||||
|
||||
break
|
||||
|
||||
return asn_data
|
||||
return response
|
||||
|
||||
except (socket.timeout, socket.error) as e: # pragma: no cover
|
||||
|
||||
|
|
@ -507,16 +422,124 @@ class Net:
|
|||
'ASN lookup failed for {0}.'.format(self.address_str)
|
||||
)
|
||||
|
||||
except ASNRegistryError:
|
||||
|
||||
raise
|
||||
|
||||
except:
|
||||
|
||||
raise ASNLookupError(
|
||||
'ASN lookup failed for {0}.'.format(self.address_str)
|
||||
)
|
||||
|
||||
def get_asn_origin_whois(self, asn_registry='radb', asn=None,
|
||||
retry_count=3, server=None, port=43):
|
||||
"""
|
||||
The function for retrieving CIDR info for an ASN via whois.
|
||||
|
||||
Args:
|
||||
asn_registry (:obj:`str`): The source to run the query against
|
||||
(asn.ASN_ORIGIN_WHOIS).
|
||||
asn (:obj:`str`): The AS number (required).
|
||||
retry_count (:obj:`int`): The number of times to retry in case
|
||||
socket errors, timeouts, connection resets, etc. are
|
||||
encountered. Defaults to 3.
|
||||
server (:obj:`str`): An optional server to connect to.
|
||||
port (:obj:`int`): The network port to connect on. Defaults to 43.
|
||||
|
||||
Returns:
|
||||
str: The raw ASN origin whois data.
|
||||
|
||||
Raises:
|
||||
WhoisLookupError: The ASN origin whois lookup failed.
|
||||
WhoisRateLimitError: The ASN origin Whois request rate limited and
|
||||
retries were exhausted.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
if server is None:
|
||||
server = ASN_ORIGIN_WHOIS[asn_registry]['server']
|
||||
|
||||
# Create the connection for the whois query.
|
||||
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
conn.settimeout(self.timeout)
|
||||
log.debug('ASN origin WHOIS query for {0} at {1}:{2}'.format(
|
||||
asn, server, port))
|
||||
conn.connect((server, port))
|
||||
|
||||
# Prep the query.
|
||||
query = ' -i origin {0}{1}'.format(asn, '\r\n')
|
||||
|
||||
# Query the whois server, and store the results.
|
||||
conn.send(query.encode())
|
||||
|
||||
response = ''
|
||||
while True:
|
||||
|
||||
d = conn.recv(4096).decode()
|
||||
|
||||
response += d
|
||||
|
||||
if not d:
|
||||
|
||||
break
|
||||
|
||||
conn.close()
|
||||
|
||||
# TODO: this was taken from get_whois(). Need to test rate limiting
|
||||
if 'Query rate limit exceeded' in response: # pragma: no cover
|
||||
|
||||
if retry_count > 0:
|
||||
|
||||
log.debug('ASN origin WHOIS query rate limit exceeded. '
|
||||
'Waiting...')
|
||||
sleep(1)
|
||||
return self.get_asn_origin_whois(
|
||||
asn_registry=asn_registry, asn=asn,
|
||||
retry_count=retry_count-1,
|
||||
server=server, port=port
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
raise WhoisRateLimitError(
|
||||
'ASN origin Whois lookup failed for {0}. Rate limit '
|
||||
'exceeded, wait and try again (possibly a '
|
||||
'temporary block).'.format(asn))
|
||||
|
||||
elif ('error 501' in response or 'error 230' in response
|
||||
): # pragma: no cover
|
||||
|
||||
log.debug('ASN origin WHOIS query error: {0}'.format(response))
|
||||
raise ValueError
|
||||
|
||||
return str(response)
|
||||
|
||||
except (socket.timeout, socket.error) as e:
|
||||
|
||||
log.debug('ASN origin WHOIS query socket error: {0}'.format(e))
|
||||
if retry_count > 0:
|
||||
|
||||
log.debug('ASN origin WHOIS query retrying (count: {0})'
|
||||
''.format(str(retry_count)))
|
||||
return self.get_asn_origin_whois(
|
||||
asn_registry=asn_registry, asn=asn,
|
||||
retry_count=retry_count-1, server=server, port=port
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
raise WhoisLookupError(
|
||||
'ASN origin WHOIS lookup failed for {0}.'.format(asn)
|
||||
)
|
||||
|
||||
except WhoisRateLimitError: # pragma: no cover
|
||||
|
||||
raise
|
||||
|
||||
except: # pragma: no cover
|
||||
|
||||
raise WhoisLookupError(
|
||||
'ASN origin WHOIS lookup failed for {0}.'.format(asn)
|
||||
)
|
||||
|
||||
def get_whois(self, asn_registry='arin', retry_count=3, server=None,
|
||||
port=43, extra_blacklist=None):
|
||||
"""
|
||||
|
|
@ -524,22 +547,26 @@ class Net:
|
|||
address via any port. Defaults to port 43/tcp (WHOIS).
|
||||
|
||||
Args:
|
||||
asn_registry: The NIC to run the query against.
|
||||
retry_count: The number of times to retry in case socket errors,
|
||||
timeouts, connection resets, etc. are encountered.
|
||||
server: An optional server to connect to. If provided, asn_registry
|
||||
will be ignored.
|
||||
port: The network port to connect on.
|
||||
extra_blacklist: A list of blacklisted whois servers in addition to
|
||||
the global BLACKLIST.
|
||||
asn_registry (:obj:`str`): The NIC to run the query against.
|
||||
Defaults to 'arin'.
|
||||
retry_count (:obj:`int`): The number of times to retry in case
|
||||
socket errors, timeouts, connection resets, etc. are
|
||||
encountered. Defaults to 3.
|
||||
server (:obj:`str`): An optional server to connect to. If
|
||||
provided, asn_registry will be ignored.
|
||||
port (:obj:`int`): The network port to connect on. Defaults to 43.
|
||||
extra_blacklist (:obj:`list` of :obj:`str`): Blacklisted whois
|
||||
servers in addition to the global BLACKLIST. Defaults to None.
|
||||
|
||||
Returns:
|
||||
String: The raw whois data.
|
||||
str: The raw whois data.
|
||||
|
||||
Raises:
|
||||
BlacklistError: Raised if the whois server provided is in the
|
||||
global BLACKLIST or extra_blacklist.
|
||||
WhoisLookupError: The whois lookup failed.
|
||||
WhoisRateLimitError: The Whois request rate limited and retries
|
||||
were exhausted.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
|
@ -585,12 +612,22 @@ class Net:
|
|||
|
||||
if 'Query rate limit exceeded' in response: # pragma: no cover
|
||||
|
||||
log.debug('WHOIS query rate limit exceeded. Waiting...')
|
||||
sleep(1)
|
||||
return self.get_whois(
|
||||
asn_registry=asn_registry, retry_count=retry_count-1,
|
||||
server=server, port=port, extra_blacklist=extra_blacklist
|
||||
)
|
||||
if retry_count > 0:
|
||||
|
||||
log.debug('WHOIS query rate limit exceeded. Waiting...')
|
||||
sleep(1)
|
||||
return self.get_whois(
|
||||
asn_registry=asn_registry, retry_count=retry_count-1,
|
||||
server=server, port=port,
|
||||
extra_blacklist=extra_blacklist
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
raise WhoisRateLimitError(
|
||||
'Whois lookup failed for {0}. Rate limit '
|
||||
'exceeded, wait and try again (possibly a '
|
||||
'temporary block).'.format(self.address_str))
|
||||
|
||||
elif ('error 501' in response or 'error 230' in response
|
||||
): # pragma: no cover
|
||||
|
|
@ -618,6 +655,10 @@ class Net:
|
|||
'WHOIS lookup failed for {0}.'.format(self.address_str)
|
||||
)
|
||||
|
||||
except WhoisRateLimitError: # pragma: no cover
|
||||
|
||||
raise
|
||||
|
||||
except BlacklistError:
|
||||
|
||||
raise
|
||||
|
|
@ -634,16 +675,18 @@ class Net:
|
|||
The function for retrieving a json result via HTTP.
|
||||
|
||||
Args:
|
||||
url: The URL to retrieve.
|
||||
retry_count: The number of times to retry in case socket errors,
|
||||
timeouts, connection resets, etc. are encountered.
|
||||
rate_limit_timeout: The number of seconds to wait before retrying
|
||||
when a rate limit notice is returned via rdap+json.
|
||||
headers: The HTTP headers dictionary. The Accept header defaults
|
||||
to 'application/rdap+json'.
|
||||
url (:obj:`str`): The URL to retrieve (required).
|
||||
retry_count (:obj:`int`): The number of times to retry in case
|
||||
socket errors, timeouts, connection resets, etc. are
|
||||
encountered. Defaults to 3.
|
||||
rate_limit_timeout (:obj:`int`): The number of seconds to wait
|
||||
before retrying when a rate limit notice is returned via
|
||||
rdap+json or HTTP error 429. Defaults to 60.
|
||||
headers (:obj:`dict`): The HTTP headers. The Accept header
|
||||
defaults to 'application/rdap+json'.
|
||||
|
||||
Returns:
|
||||
Dictionary: The data in json format.
|
||||
dict: The data in json format.
|
||||
|
||||
Raises:
|
||||
HTTPLookupError: The HTTP lookup failed.
|
||||
|
|
@ -695,17 +738,36 @@ class Net:
|
|||
|
||||
return d
|
||||
|
||||
except HTTPError as e: # pragma: no cover
|
||||
|
||||
# RIPE is producing this HTTP error rather than a JSON error.
|
||||
if e.code == 429:
|
||||
|
||||
log.debug('HTTP query rate limit exceeded.')
|
||||
|
||||
if retry_count > 0:
|
||||
log.debug('Waiting {0} seconds...'.format(
|
||||
str(rate_limit_timeout)))
|
||||
|
||||
sleep(rate_limit_timeout)
|
||||
return self.get_http_json(
|
||||
url=url, retry_count=retry_count - 1,
|
||||
rate_limit_timeout=rate_limit_timeout,
|
||||
headers=headers
|
||||
)
|
||||
else:
|
||||
raise HTTPRateLimitError(
|
||||
'HTTP lookup failed for {0}. Rate limit '
|
||||
'exceeded, wait and try again (possibly a '
|
||||
'temporary block).'.format(url))
|
||||
|
||||
else:
|
||||
|
||||
raise HTTPLookupError('HTTP lookup failed for {0} with error '
|
||||
'code {1}.'.format(url, str(e.code)))
|
||||
|
||||
except (URLError, socket.timeout, socket.error) as e:
|
||||
|
||||
# Check needed for Python 2.6, also why URLError is caught.
|
||||
try: # pragma: no cover
|
||||
if not isinstance(e.reason, (socket.timeout, socket.error)):
|
||||
raise HTTPLookupError('HTTP lookup failed for {0}.'
|
||||
''.format(url))
|
||||
except AttributeError: # pragma: no cover
|
||||
|
||||
pass
|
||||
|
||||
log.debug('HTTP query socket error: {0}'.format(e))
|
||||
if retry_count > 0:
|
||||
|
||||
|
|
@ -735,11 +797,17 @@ class Net:
|
|||
The function for retrieving host information for an IP address.
|
||||
|
||||
Args:
|
||||
retry_count: The number of times to retry in case socket errors,
|
||||
timeouts, connection resets, etc. are encountered.
|
||||
retry_count (:obj:`int`): The number of times to retry in case
|
||||
socket errors, timeouts, connection resets, etc. are
|
||||
encountered. Defaults to 3.
|
||||
|
||||
Returns:
|
||||
Tuple: hostname, aliaslist, ipaddrlist
|
||||
namedtuple:
|
||||
|
||||
:hostname (str): The hostname returned mapped to the given IP
|
||||
address.
|
||||
:aliaslist (list): Alternate names for the given IP address.
|
||||
:ipaddrlist (list): IPv4/v6 addresses mapped to the same hostname.
|
||||
|
||||
Raises:
|
||||
HostLookupError: The host lookup failed.
|
||||
|
|
@ -760,7 +828,9 @@ class Net:
|
|||
|
||||
socket.setdefaulttimeout(None)
|
||||
|
||||
return ret
|
||||
results = namedtuple('get_host_results', 'hostname, aliaslist, '
|
||||
'ipaddrlist')
|
||||
return results(ret)
|
||||
|
||||
except (socket.timeout, socket.error) as e:
|
||||
|
||||
|
|
@ -784,110 +854,24 @@ class Net:
|
|||
'Host lookup failed for {0}.'.format(self.address_str)
|
||||
)
|
||||
|
||||
def lookup_asn(self, retry_count=3, asn_alts=None, extra_org_map=None):
|
||||
"""
|
||||
The wrapper function for retrieving and parsing ASN information for an
|
||||
IP address.
|
||||
|
||||
Args:
|
||||
retry_count: The number of times to retry in case socket errors,
|
||||
timeouts, connection resets, etc. are encountered.
|
||||
asn_alts: Array of additional lookup types to attempt if the
|
||||
ASN dns lookup fails. Allow permutations must be enabled.
|
||||
Defaults to all ['whois', 'http'].
|
||||
extra_org_map: Dictionary mapping org handles to RIRs. This is for
|
||||
limited cases where ARIN REST (ASN fallback HTTP lookup) does
|
||||
not show an RIR as the org handle e.g., DNIC (which is now the
|
||||
built in ORG_MAP) e.g., {'DNIC': 'arin'}. Valid RIR values are
|
||||
(note the case-sensitive - this is meant to match the REST
|
||||
result): 'ARIN', 'RIPE', 'apnic', 'lacnic', 'afrinic'
|
||||
|
||||
Returns:
|
||||
Tuple:
|
||||
|
||||
:Dictionary: Result from get_asn_dns() or get_asn_whois().
|
||||
:Dictionary: The response returned by get_asn_dns() or
|
||||
get_asn_whois().
|
||||
|
||||
Raises:
|
||||
ASNRegistryError: ASN registry does not match.
|
||||
HTTPLookupError: The HTTP lookup failed.
|
||||
"""
|
||||
|
||||
lookups = asn_alts if asn_alts is not None else ['whois', 'http']
|
||||
|
||||
# Initialize the response.
|
||||
response = None
|
||||
|
||||
# Attempt to resolve ASN info via Cymru. DNS is faster, try that first.
|
||||
try:
|
||||
|
||||
self.dns_resolver.lifetime = self.dns_resolver.timeout * (
|
||||
retry_count and retry_count or 1)
|
||||
asn_data = self.get_asn_dns()
|
||||
|
||||
except (ASNLookupError, ASNRegistryError) as e:
|
||||
|
||||
if not self.allow_permutations:
|
||||
|
||||
raise ASNRegistryError('ASN registry lookup failed. '
|
||||
'Permutations not allowed.')
|
||||
|
||||
try:
|
||||
if 'whois' in lookups:
|
||||
|
||||
log.debug('ASN DNS lookup failed, trying ASN WHOIS: '
|
||||
'{0}'.format(e))
|
||||
asn_data = self.get_asn_whois(retry_count)
|
||||
|
||||
else:
|
||||
|
||||
raise ASNLookupError
|
||||
|
||||
except (ASNLookupError, ASNRegistryError): # pragma: no cover
|
||||
|
||||
if 'http' in lookups:
|
||||
|
||||
# Lets attempt to get the ASN registry information from
|
||||
# ARIN.
|
||||
log.debug('ASN WHOIS lookup failed, trying ASN via HTTP')
|
||||
try:
|
||||
|
||||
asn_data = self.get_asn_http(
|
||||
retry_count=retry_count,
|
||||
extra_org_map=extra_org_map
|
||||
)
|
||||
|
||||
except ASNRegistryError:
|
||||
|
||||
raise ASNRegistryError('ASN registry lookup failed.')
|
||||
|
||||
except ASNLookupError:
|
||||
|
||||
raise HTTPLookupError('ASN HTTP lookup failed.')
|
||||
|
||||
else:
|
||||
|
||||
raise ASNRegistryError('ASN registry lookup failed.')
|
||||
|
||||
return asn_data, response
|
||||
|
||||
def get_http_raw(self, url=None, retry_count=3, headers=None,
|
||||
request_type='GET', form_data=None):
|
||||
"""
|
||||
The function for retrieving a raw HTML result via HTTP.
|
||||
|
||||
Args:
|
||||
url: The URL to retrieve.
|
||||
retry_count: The number of times to retry in case socket errors,
|
||||
timeouts, connection resets, etc. are encountered.
|
||||
headers: The HTTP headers dictionary. The Accept header defaults
|
||||
to 'application/rdap+json'.
|
||||
request_type: 'GET' or 'POST'
|
||||
form_data: Dictionary of form POST data
|
||||
url (:obj:`str`): The URL to retrieve (required).
|
||||
retry_count (:obj:`int`): The number of times to retry in case
|
||||
socket errors, timeouts, connection resets, etc. are
|
||||
encountered. Defaults to 3.
|
||||
headers (:obj:`dict`): The HTTP headers. The Accept header
|
||||
defaults to 'text/html'.
|
||||
request_type (:obj:`str`): Request type 'GET' or 'POST'. Defaults
|
||||
to 'GET'.
|
||||
form_data (:obj:`dict`): Optional form POST data.
|
||||
|
||||
Returns:
|
||||
String: The raw data.
|
||||
str: The raw data.
|
||||
|
||||
Raises:
|
||||
HTTPLookupError: The HTTP lookup failed.
|
||||
|
|
@ -896,10 +880,12 @@ class Net:
|
|||
if headers is None:
|
||||
headers = {'Accept': 'text/html'}
|
||||
|
||||
enc_form_data = None
|
||||
if form_data:
|
||||
form_data = urlencode(form_data)
|
||||
enc_form_data = urlencode(form_data)
|
||||
try:
|
||||
form_data = bytes(form_data, encoding='ascii')
|
||||
# Py 2 inspection will alert on the encoding arg, no harm done.
|
||||
enc_form_data = bytes(enc_form_data, encoding='ascii')
|
||||
except TypeError: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
|
@ -909,10 +895,11 @@ class Net:
|
|||
log.debug('HTTP query for {0} at {1}'.format(
|
||||
self.address_str, url))
|
||||
try:
|
||||
conn = Request(url=url, data=form_data, headers=headers,
|
||||
method=request_type)
|
||||
# Py 2 inspection alert bypassed by using kwargs dict.
|
||||
conn = Request(url=url, data=enc_form_data, headers=headers,
|
||||
**{'method': request_type})
|
||||
except TypeError: # pragma: no cover
|
||||
conn = Request(url=url, data=form_data, headers=headers)
|
||||
conn = Request(url=url, data=enc_form_data, headers=headers)
|
||||
data = self.opener.open(conn, timeout=self.timeout)
|
||||
|
||||
try:
|
||||
|
|
@ -924,15 +911,6 @@ class Net:
|
|||
|
||||
except (URLError, socket.timeout, socket.error) as e:
|
||||
|
||||
# Check needed for Python 2.6, also why URLError is caught.
|
||||
try: # pragma: no cover
|
||||
if not isinstance(e.reason, (socket.timeout, socket.error)):
|
||||
raise HTTPLookupError('HTTP lookup failed for {0}.'
|
||||
''.format(url))
|
||||
except AttributeError: # pragma: no cover
|
||||
|
||||
pass
|
||||
|
||||
log.debug('HTTP query socket error: {0}'.format(e))
|
||||
if retry_count > 0:
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue