From d419823b49c3ba781c88cedf72d5bc51fa47e4b5 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Fri, 19 Oct 2018 13:08:26 +0100 Subject: [PATCH 01/36] :bug: Regex not parsing response in `asn.py` The regex expression was not parsing the response correctly in `asn.py:get_nets_radb`. May issue seems to be the ipwhois data may have changed no long ends in
instead with \n. --- ipwhois/asn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 6f15fa1..0ce8ef9 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -740,9 +740,9 @@ class ASNOrigin: nets = [] if is_http: - regex = r'route(?:6)?:[^\S\n]+(?P.+?)
' + regex = r'route(?:6)?:\s+(?P.+?).?\n' else: - regex = r'^route(?:6)?:[^\S\n]+(?P.+|.+)$' + regex = r'^route(?:6)?:\s+(?P.+|.+)$' # Iterate through all of the networks found, storing the CIDR value # and the start and end positions. From 48619682d0ef1356a046d2c3fddf91ceaa821161 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Fri, 19 Oct 2018 14:35:46 +0100 Subject: [PATCH 02/36] :white_check_mark: Fixed unit test I think unit test should be passing, fixed unit test so it passes. The following test test_asn:test__get_nets_radb was asserting an empty list [], but I think the data should be parseable. --- ipwhois/asn.py | 4 ++-- ipwhois/tests/test_asn.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 0ce8ef9..9babae1 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -740,9 +740,9 @@ class ASNOrigin: nets = [] if is_http: - regex = r'route(?:6)?:\s+(?P.+?).?\n' + regex = r'route(?:6)?:[^\S\n]+(?P.+?)\n' else: - regex = r'^route(?:6)?:\s+(?P.+|.+)$' + regex = r'^route(?:6)?:[^\S\n]+(?P.+|.+)$' # Iterate through all of the networks found, storing the CIDR value # and the start and end positions. diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index 5bdb134..0485abb 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -242,7 +242,10 @@ class TestASNOrigin(TestCommon): obj._get_nets_radb(multi_net_response) self.assertEqual(obj._get_nets_radb(multi_net_response, is_http=True), - []) + [{'cidr': '66.249.64.0/20', 'description': None, 'maintainer': None, 'updated': None, + 'source': None, 'start': 2, 'end': 29}, + {'cidr': '66.249.80.0/20', 'description': None, 'maintainer': None, 'updated': None, + 'source': None, 'start': 175, 'end': 202}]) net = Net('2001:43f8:7b0::') obj = ASNOrigin(net) From 600b4b09e47ab913c4841dac0d5a14a735ba989f Mon Sep 17 00:00:00 2001 From: secynic Date: Fri, 1 Feb 2019 23:59:44 -0600 Subject: [PATCH 03/36] remove deprecated (#230) --- ASN.rst | 12 ---- CHANGES.rst | 13 ++++ CLI.rst | 8 +-- RDAP.rst | 6 -- README.rst | 5 -- UPGRADING.rst | 14 +++++ WHOIS.rst | 6 -- ipwhois/asn.py | 91 ++-------------------------- ipwhois/ipwhois.py | 25 ++------ ipwhois/net.py | 31 +--------- ipwhois/nir.py | 40 ------------ ipwhois/scripts/ipwhois_cli.py | 17 ------ ipwhois/tests/online/test_ipwhois.py | 5 -- ipwhois/whois.py | 40 ------------ 14 files changed, 41 insertions(+), 272 deletions(-) diff --git a/ASN.rst b/ASN.rst index f2ef8f1..bcfed90 100644 --- a/ASN.rst +++ b/ASN.rst @@ -24,12 +24,6 @@ Arguments supported by IPASN.lookup(). | | | resets, etc. are encountered. | | | | Defaults to 3. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN | | | | REST (ASN fallback HTTP lookup) does not | @@ -157,12 +151,6 @@ Arguments supported by ASNOrigin.lookup(). | | | ['description', 'maintainer', 'updated', | | | | 'source']. If None, defaults to all. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | asn_methods | list | ASN lookup types to attempt, in order. If | | | | None, defaults to all ['whois', 'http']. | +------------------------+--------+-------------------------------------------+ diff --git a/CHANGES.rst b/CHANGES.rst index 0a2ec4f..024080b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,19 @@ Changelog ========= +1.2.0 (TBD) +------------------ + +- Removed deprecated functions: asn.IPASN._parse_fields_http, + asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois, + asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb, + net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin + whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other, + nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic + nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact (#230) +- Removed deprecated asn_alts parameter (#230) +- Removed deprecated allow_permutations parameter (#230) + 1.1.0 (2019-02-01) ------------------ diff --git a/CLI.rst b/CLI.rst index cf4a986..03902a2 100644 --- a/CLI.rst +++ b/CLI.rst @@ -22,7 +22,7 @@ ipwhois_cli.py [-h] [--whois] [--exclude_nir] [--json] [--hr] [--proxy_http "PROXY_HTTP"] [--proxy_https "PROXY_HTTPS"] [--inc_raw] [--retry_count RETRY_COUNT] - [--asn_alts "ASN_ALTS"] [--asn_methods "ASN_METHODS"] + [--asn_methods "ASN_METHODS"] [--extra_org_map "EXTRA_ORG_MAP"] [--skip_asn_description] [--depth COLOR_DEPTH] [--excluded_entities "EXCLUDED_ENTITIES"] [--bootstrap] @@ -66,12 +66,6 @@ Common settings (RDAP & Legacy Whois): --retry_count RETRY_COUNT The number of times to retry in case socket errors, timeouts, connection resets, etc. are encountered. - --asn_alts ASN_ALTS - A comma delimited list of additional lookup types to - attempt if the ASN dns lookup fails. Allow - permutations must be enabled. Defaults to all: - "whois,http". *WARNING* deprecated in favor of new - argument asn_methods. --asn_methods ASN_METHODS List of ASN lookup types to attempt, in order. Defaults to all ['dns', 'whois', 'http']. diff --git a/RDAP.rst b/RDAP.rst index f216b3a..9d32b50 100644 --- a/RDAP.rst +++ b/RDAP.rst @@ -42,12 +42,6 @@ Arguments supported by IPWhois.lookup_rdap(). | | | when a rate limit notice is returned via | | | | rdap+json. Defaults to 120. | +--------------------+--------+-----------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the ASN | -| | | dns lookup fails. Allow permutations must be | -| | | enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated in | -| | | favor of new argument asn_methods. | -+--------------------+--------+-----------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN REST | | | | (ASN fallback HTTP lookup) does not show an | diff --git a/README.rst b/README.rst index eca1a08..712c939 100644 --- a/README.rst +++ b/README.rst @@ -170,11 +170,6 @@ Input | proxy_opener | object | The urllib.request.OpenerDirector request for | | | | proxy support or None. | +--------------------+--------+-----------------------------------------------+ -| allow_permutations | bool | Allow net.Net() to use additional methods if | -| | | DNS lookups to Cymru fail. *WARNING* | -| | | deprecated in favor of new argument | -| | | asn_methods. Defaults to False. | -+--------------------+--------+-----------------------------------------------+ RDAP (HTTP) ----------- diff --git a/UPGRADING.rst b/UPGRADING.rst index 679db1a..d26f5f2 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -9,6 +9,20 @@ any changes that may affect user experience when upgrading to a new release. This page is new as of version 1.0.0. Any information on older versions is likely missing or incomplete. +****** +v1.2.0 +****** + +- Removed deprecated functions: asn.IPASN._parse_fields_http, + asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois, + asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb, + net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin + whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other, + nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic + nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact +- Removed deprecated asn_alts parameter +- Removed deprecated allow_permutations parameter + ****** v1.1.0 ****** diff --git a/WHOIS.rst b/WHOIS.rst index bb7a9b0..0d7b496 100644 --- a/WHOIS.rst +++ b/WHOIS.rst @@ -53,12 +53,6 @@ Arguments supported by IPWhois.lookup_whois(). | | | 'postal_code', 'emails', 'created', | | | | 'updated']. If None, defaults to all. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN | | | | REST (ASN fallback HTTP lookup) does not | diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 03946fe..d626662 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -169,16 +169,6 @@ class IPASN: return ret - def _parse_fields_dns(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_dns() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_dns().') - return self.parse_fields_dns(*args, **kwargs) - def parse_fields_verbose_dns(self, response): """ The function for parsing ASN fields from a verbose dns response. @@ -293,16 +283,6 @@ class IPASN: return ret - def _parse_fields_whois(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_whois() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_whois().') - return self.parse_fields_whois(*args, **kwargs) - def parse_fields_http(self, response, extra_org_map=None): """ The function for parsing ASN fields from a http response. @@ -403,19 +383,8 @@ class IPASN: return asn_data - def _parse_fields_http(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_http() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_http().') - return self.parse_fields_http(*args, **kwargs) - - def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, - extra_org_map=None, asn_methods=None, - get_asn_description=True): + def lookup(self, inc_raw=False, retry_count=3, extra_org_map=None, + asn_methods=None, get_asn_description=True): """ The wrapper function for retrieving and parsing ASN information for an IP address. @@ -426,10 +395,6 @@ class IPASN: retry_count (:obj:`int`): The number of times to retry in case socket errors, timeouts, connection resets, etc. are encountered. Defaults to 3. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - Defaults to all ['whois', 'http']. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to None. extra_org_map (:obj:`dict`): Mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC (which is @@ -466,17 +431,7 @@ class IPASN: if asn_methods is None: - if asn_alts is None: - - lookups = ['dns', 'whois', 'http'] - - else: - - from warnings import warn - warn('IPASN.lookup() asn_alts argument has been deprecated ' - 'and will be removed. You should now use the asn_methods ' - 'argument.') - lookups = ['dns'] + asn_alts + lookups = ['dns', 'whois', 'http'] else: @@ -492,8 +447,7 @@ class IPASN: dns_success = False for index, lookup_method in enumerate(lookups): - if index > 0 and not asn_methods and not ( - self._net.allow_permutations): + if index > 0 and not asn_methods: raise ASNRegistryError('ASN registry lookup failed. ' 'Permutations not allowed.') @@ -706,16 +660,6 @@ class ASNOrigin: return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('ASNOrigin._parse_fields() has been deprecated and will be ' - 'removed. You should now use ASNOrigin.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_radb(self, response, is_http=False): """ The function for parsing network blocks from ASN origin data. @@ -769,18 +713,8 @@ class ASNOrigin: return nets - def _get_nets_radb(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('ASNOrigin._get_nets_radb() has been deprecated and will be ' - 'removed. You should now use ASNOrigin.get_nets_radb().') - return self.get_nets_radb(*args, **kwargs) - def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, - field_list=None, asn_alts=None, asn_methods=None): + field_list=None, asn_methods=None): """ The function for retrieving and parsing ASN origin whois information via port 43/tcp (WHOIS). @@ -797,9 +731,6 @@ class ASNOrigin: field_list (:obj:`list`): If provided, fields to parse: ['description', 'maintainer', 'updated', 'source'] If None, defaults to all. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN whois lookup fails. If None, defaults to all ['http']. - *WARNING* deprecated in favor of new argument asn_methods. asn_methods (:obj:`list`): ASN lookup types to attempt, in order. If None, defaults to all ['whois', 'http']. @@ -828,17 +759,7 @@ class ASNOrigin: if asn_methods is None: - if asn_alts is None: - - lookups = ['whois', 'http'] - - else: - - from warnings import warn - warn('ASNOrigin.lookup() asn_alts argument has been deprecated' - ' and will be removed. You should now use the asn_methods' - ' argument.') - lookups = ['whois'] + asn_alts + lookups = ['whois', 'http'] else: diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index 4a4d56b..0c3e179 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -42,17 +42,12 @@ class IPWhois: seconds. Defaults to 5. proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for proxy support. Defaults to None. - allow_permutations (:obj:`bool`): Allow net.Net() to use additional - methods if DNS lookups to Cymru fail. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to False. """ - def __init__(self, address, timeout=5, proxy_opener=None, - allow_permutations=False): + def __init__(self, address, timeout=5, proxy_opener=None): self.net = Net( - address=address, timeout=timeout, proxy_opener=proxy_opener, - allow_permutations=allow_permutations + address=address, timeout=timeout, proxy_opener=proxy_opener ) self.ipasn = IPASN(self.net) @@ -71,7 +66,7 @@ class IPWhois: def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, extra_blacklist=None, ignore_referral_errors=False, - field_list=None, asn_alts=None, extra_org_map=None, + field_list=None, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, get_asn_description=True): """ @@ -95,10 +90,6 @@ class IPWhois: ['name', 'handle', 'description', 'country', 'state', 'city', 'address', 'postal_code', 'emails', 'created', 'updated'] If None, defaults to all. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - If None, defaults to all ['whois', 'http']. *WARNING* - deprecated in favor of new argument asn_methods. extra_org_map (:obj:`dict`): Dictionary mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC @@ -161,7 +152,7 @@ class IPWhois: log.debug('ASN lookup for {0}'.format(self.address_str)) asn_data = self.ipasn.lookup( - inc_raw=inc_raw, retry_count=retry_count, asn_alts=asn_alts, + inc_raw=inc_raw, retry_count=retry_count, extra_org_map=extra_org_map, asn_methods=asn_methods, get_asn_description=get_asn_description ) @@ -206,7 +197,7 @@ class IPWhois: def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, excluded_entities=None, bootstrap=False, - rate_limit_timeout=120, asn_alts=None, extra_org_map=None, + rate_limit_timeout=120, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, get_asn_description=True): """ @@ -233,10 +224,6 @@ class IPWhois: rate_limit_timeout (:obj:`int`): The number of seconds to wait before retrying when a rate limit notice is returned via rdap+json. Defaults to 120. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - If None, defaults to all ['whois', 'http']. *WARNING* - deprecated in favor of new argument asn_methods. extra_org_map (:obj:`dict`): Dictionary mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC @@ -303,7 +290,7 @@ class IPWhois: # Retrieve the ASN information. log.debug('ASN lookup for {0}'.format(self.address_str)) asn_data = self.ipasn.lookup( - inc_raw=inc_raw, retry_count=retry_count, asn_alts=asn_alts, + inc_raw=inc_raw, retry_count=retry_count, extra_org_map=extra_org_map, asn_methods=asn_methods, get_asn_description=get_asn_description ) diff --git a/ipwhois/net.py b/ipwhois/net.py index 8793921..ac23c35 100644 --- a/ipwhois/net.py +++ b/ipwhois/net.py @@ -103,17 +103,13 @@ class Net: seconds. Defaults to 5. proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for proxy support. Defaults to None. - allow_permutations (:obj:`bool`): Allow net.Net() to use additional - methods if DNS lookups to Cymru fail. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to False. Raises: IPDefinedError: The address provided is defined (does not need to be resolved). """ - def __init__(self, address, timeout=5, proxy_opener=None, - allow_permutations=False): + def __init__(self, address, timeout=5, proxy_opener=None): # IPv4Address or IPv6Address if isinstance(address, IPv4Address) or isinstance( @@ -129,16 +125,6 @@ class Net: # Default timeout for socket connections. self.timeout = timeout - # Allow other than DNS lookups for ASNs. - self.allow_permutations = allow_permutations - - if self.allow_permutations: - - from warnings import warn - warn('allow_permutations has been deprecated and will be removed. ' - 'It is no longer needed, due to the deprecation of asn_alts, ' - 'and the addition of the asn_methods argument.') - self.dns_resolver = dns.resolver.Resolver() self.dns_resolver.timeout = timeout self.dns_resolver.lifetime = timeout @@ -219,21 +205,6 @@ class Net: self.dns_zone = IPV6_DNS_ZONE.format(self.reversed) - def lookup_asn(self, *args, **kwargs): - """ - Temporary wrapper for IP ASN lookups (moved to - asn.IPASN.lookup()). This will be removed in a future - release. - """ - - from warnings import warn - warn('Net.lookup_asn() has been deprecated and will be removed. ' - 'You should now use asn.IPASN.lookup() for IP ASN lookups.') - from .asn import IPASN - response = None - ipasn = IPASN(self) - return ipasn.lookup(*args, **kwargs), response - def get_asn_dns(self): """ The function for retrieving ASN information for an IP address from diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 910b4f8..2c470ac 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -286,16 +286,6 @@ class NIRWhois: return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._parse_fields() has been deprecated and will be ' - 'removed. You should now use NIRWhois.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_jpnic(self, response): """ The function for parsing network blocks from jpnic whois data. @@ -359,16 +349,6 @@ class NIRWhois: return nets - def _get_nets_jpnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_nets_jpnic() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_nets_jpnic().') - return self.get_nets_jpnic(*args, **kwargs) - def get_nets_krnic(self, response): """ The function for parsing network blocks from krnic whois data. @@ -434,16 +414,6 @@ class NIRWhois: return nets - def _get_nets_krnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_nets_krnic() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_nets_krnic().') - return self.get_nets_krnic(*args, **kwargs) - def get_contact(self, response=None, nir=None, handle=None, retry_count=3, dt_format=None): """ @@ -491,16 +461,6 @@ class NIRWhois: is_contact=True ) - def _get_contact(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_contact() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_contact().') - return self.get_contact(*args, **kwargs) - def lookup(self, nir=None, inc_raw=False, retry_count=3, response=None, field_list=None, is_offline=False): """ diff --git a/ipwhois/scripts/ipwhois_cli.py b/ipwhois/scripts/ipwhois_cli.py index 5a06e9a..74a08f9 100644 --- a/ipwhois/scripts/ipwhois_cli.py +++ b/ipwhois/scripts/ipwhois_cli.py @@ -166,17 +166,6 @@ group.add_argument( help='The number of times to retry in case socket errors, timeouts, ' 'connection resets, etc. are encountered.' ) -group.add_argument( - '--asn_alts', - type=str, - nargs=1, - default='whois,http', - metavar='"ASN_ALTS"', - help='A comma delimited list of additional lookup types to attempt if the ' - 'ASN dns lookup fails. Allow permutations must be enabled. ' - 'Defaults to all: "whois,http" *WARNING* deprecated in ' - 'favor of new argument asn_methods.' -) group.add_argument( '--asn_methods', type=str, @@ -1456,9 +1445,6 @@ if script_args.addr: field_list=script_args.field_list.split(',') if ( script_args.field_list and len(script_args.field_list) > 0) else None, - asn_alts=script_args.asn_alts.split(',') if ( - script_args.asn_alts and not script_args.asn_methods and - len(script_args.asn_alts) > 0) else None, extra_org_map=script_args.extra_org_map, inc_nir=(not script_args.exclude_nir), nir_field_list=script_args.nir_field_list.split(',') if ( @@ -1484,9 +1470,6 @@ if script_args.addr: len(script_args.excluded_entities) > 0) else None, bootstrap=script_args.bootstrap, rate_limit_timeout=script_args.rate_limit_timeout, - asn_alts=script_args.asn_alts.split(',') if ( - script_args.asn_alts and not script_args.asn_methods and - len(script_args.asn_alts) > 0) else None, extra_org_map=script_args.extra_org_map, inc_nir=(not script_args.exclude_nir), nir_field_list=script_args.nir_field_list.split(',') if ( diff --git a/ipwhois/tests/online/test_ipwhois.py b/ipwhois/tests/online/test_ipwhois.py index d9f24a7..3627e9d 100644 --- a/ipwhois/tests/online/test_ipwhois.py +++ b/ipwhois/tests/online/test_ipwhois.py @@ -174,8 +174,3 @@ class TestIPWhois(TestCommon): result = IPWhois(address='74.125.225.229', timeout=0, proxy_opener=opener) self.assertRaises(ASNRegistryError, result.lookup_rdap) - - log.debug('Testing allow_permutations') - result = IPWhois(address='74.125.225.229', timeout=0, - allow_permutations=False) - self.assertRaises(ASNRegistryError, result.lookup_rdap) diff --git a/ipwhois/whois.py b/ipwhois/whois.py index 882dd90..abc3361 100644 --- a/ipwhois/whois.py +++ b/ipwhois/whois.py @@ -324,16 +324,6 @@ class Whois: return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._parse_fields() has been deprecated and will be ' - 'removed. You should now use Whois.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_arin(self, response): """ The function for parsing network blocks from ARIN whois data. @@ -415,16 +405,6 @@ class Whois: return nets - def _get_nets_arin(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_arin() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_arin().') - return self.get_nets_arin(*args, **kwargs) - def get_nets_lacnic(self, response): """ The function for parsing network blocks from LACNIC whois data. @@ -495,16 +475,6 @@ class Whois: return nets - def _get_nets_lacnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_lacnic() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_lacnic().') - return self.get_nets_lacnic(*args, **kwargs) - def get_nets_other(self, response): """ The function for parsing network blocks from generic whois data. @@ -577,16 +547,6 @@ class Whois: return nets - def _get_nets_other(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_other() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_other().') - return self.get_nets_other(*args, **kwargs) - def lookup(self, inc_raw=False, retry_count=3, response=None, get_referral=False, extra_blacklist=None, ignore_referral_errors=False, asn_data=None, From aa2de4a3e152faf132f30b2dea9556d5f37a981b Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Mon, 4 Feb 2019 13:55:38 +0000 Subject: [PATCH 04/36] 100% Coverage Added tests to make 100% code coverage again. Also updated coveragerc to show missing lines in the coverage report. --- .coveragerc | 1 + ipwhois/tests/test_asn.py | 26 ++++++++++++++++++++++++-- ipwhois/tests/test_nir.py | 5 +++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 5f6fa55..3aab48d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [report] +show_missing = True omit = */python?.?/* */site-packages/nose/* diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index 0485abb..1981de2 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -135,8 +135,30 @@ class TestIPASN(TestCommon): self.fail('Unexpected exception raised: {0}'.format(e)) def test_lookup(self): - # TODO: need to modify asn.json for this. - return NotImplemented + data_dir = path.dirname(__file__) + + with io.open(str(data_dir) + '/asn.json', 'r') as \ + data_file: + data = json.load(data_file) + + for key, val in data.items(): + + log.debug('Testing: {0}'.format(key)) + net = Net(key) + obj = IPASN(net) + + try: + + self.assertIsInstance(obj.lookup(), dict) + self.assertIsInstance(obj.lookup(asn_alts=['http']), dict) + + except AssertionError as e: + + raise e + + except Exception as e: + + self.fail('Unexpected exception raised: {0}'.format(e)) class TestASNOrigin(TestCommon): diff --git a/ipwhois/tests/test_nir.py b/ipwhois/tests/test_nir.py index a855e82..9220309 100644 --- a/ipwhois/tests/test_nir.py +++ b/ipwhois/tests/test_nir.py @@ -45,6 +45,11 @@ class TestNIRWhois(TestCommon): inc_raw=True), dict) + self.assertIsInstance(obj.lookup( + nir=val['nir'], + response=val['response']), + dict) + except AssertionError as e: raise e From e947df634b55ec99a9ce141354327749724292f3 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Tue, 5 Feb 2019 11:14:46 +0000 Subject: [PATCH 05/36] Changes from secynic Changes from secynic. --- ipwhois/tests/test_asn.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index b21249f..f83cce5 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -136,30 +136,7 @@ class TestIPASN(TestCommon): self.fail('Unexpected exception raised: {0}'.format(e)) def test_lookup(self): - data_dir = path.dirname(__file__) - - with io.open(str(data_dir) + '/asn.json', 'r') as \ - data_file: - data = json.load(data_file) - - for key, val in data.items(): - - log.debug('Testing: {0}'.format(key)) - net = Net(key) - obj = IPASN(net) - - try: - - self.assertIsInstance(obj.lookup(), dict) - self.assertIsInstance(obj.lookup(asn_alts=['http']), dict) - - except AssertionError as e: - - raise e - - except Exception as e: - - self.fail('Unexpected exception raised: {0}'.format(e)) + raise NotImplemented class TestASNOrigin(TestCommon): From e20c06e1d3ac9fc6ca72aaad51718f0c709be9f8 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Mon, 11 Feb 2019 11:57:01 +0000 Subject: [PATCH 06/36] Code Review Changes Changes to test as suggest by the maintainer --- ipwhois/tests/test_asn.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index f83cce5..9f9425a 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -136,7 +136,29 @@ class TestIPASN(TestCommon): self.fail('Unexpected exception raised: {0}'.format(e)) def test_lookup(self): - raise NotImplemented + data_dir = path.dirname(__file__) + + with io.open(str(data_dir) + '/asn.json', 'r') as \ + data_file: + data = json.load(data_file) + + for key, val in data.items(): + + log.debug('Testing: {0}'.format(key)) + net = Net(key) + obj = IPASN(net) + + try: + + self.assertIsInstance(obj.lookup(), dict) + + except AssertionError as e: + + raise e + + except Exception as e: + + self.fail('Unexpected exception raised: {0}'.format(e)) class TestASNOrigin(TestCommon): From b27953ca79a03c17d5f73a89ad331cedebebaa37 Mon Sep 17 00:00:00 2001 From: Stanislav Levin Date: Sat, 16 Feb 2019 20:17:56 +0300 Subject: [PATCH 07/36] Make use of builtin sphinx napoleon extension Since 1.3b1 (released Oct 10, 2014) Sphinx has support for NumPy and Google style docstring support via sphinx.ext.napoleon extension. The sphinxcontrib.napoleon extension will continue to work with Sphinx <= 1.2. Signed-off-by: Stanislav Levin --- ipwhois/docs/requirements.txt | 1 - ipwhois/docs/source/conf.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ipwhois/docs/requirements.txt b/ipwhois/docs/requirements.txt index 8aba4d0..b7845ba 100644 --- a/ipwhois/docs/requirements.txt +++ b/ipwhois/docs/requirements.txt @@ -1,4 +1,3 @@ sphinx -sphinxcontrib-napoleon sphinx_rtd_theme dnspython diff --git a/ipwhois/docs/source/conf.py b/ipwhois/docs/source/conf.py index 164be02..108ae4d 100644 --- a/ipwhois/docs/source/conf.py +++ b/ipwhois/docs/source/conf.py @@ -40,7 +40,7 @@ extensions = [ 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', - 'sphinxcontrib.napoleon' + 'sphinx.ext.napoleon' ] napoleon_google_docstring = True From 3b11faa8da71b22ba7e78e1291d1b818992e0dbf Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Fri, 17 May 2019 14:05:56 +0100 Subject: [PATCH 08/36] Updated gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 690d6be..8b4d293 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,7 @@ nosetests.xml .pydevproject MANIFEST -.idea \ No newline at end of file +.idea +.history +.vscode +.venv \ No newline at end of file From 153c402be48a0c9119ac2cfe39128a0fb05b4fb2 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Tue, 30 Jul 2019 10:18:18 +0100 Subject: [PATCH 09/36] Fixed changes Fixed changes from code review. --- ipwhois/tests/test_asn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index 9f9425a..bb73051 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -145,8 +145,8 @@ class TestIPASN(TestCommon): for key, val in data.items(): log.debug('Testing: {0}'.format(key)) - net = Net(key) - obj = IPASN(net) + net = Net(key) + obj = IPASN(net) try: From 566dc3e280a57ac6db1259dd027352223ca1b036 Mon Sep 17 00:00:00 2001 From: secynic Date: Wed, 31 Jul 2019 13:29:11 -0500 Subject: [PATCH 10/36] Fixed ASNOrigin lookups (#216) --- CHANGES.rst | 2 ++ ipwhois/asn.py | 31 +++++++++++++++++++------------ ipwhois/tests/online/test_asn.py | 3 ++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 024080b..663cb00 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,8 @@ Changelog nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact (#230) - Removed deprecated asn_alts parameter (#230) - Removed deprecated allow_permutations parameter (#230) +- Fixed ASNOrigin lookups (#216) +- Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216) 1.1.0 (2019-02-01) ------------------ diff --git a/ipwhois/asn.py b/ipwhois/asn.py index aaa4c33..60625e2 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -61,21 +61,21 @@ ASN_ORIGIN_WHOIS = { ASN_ORIGIN_HTTP = { 'radb': { - 'url': 'http://www.radb.net/query/', + 'url': 'http://www.radb.net/query', 'form_data_asn_field': 'keywords', 'form_data': { 'advanced_query': '1', 'query': 'Query', - '-T option': 'inet-rtr', + # '-T option': 'inet-rtr', 'ip_option': '', '-i': '1', '-i option': 'origin' }, 'fields': { - 'description': r'(descr):[^\S\n]+(?P.+?)\', - 'maintainer': r'(mnt-by):[^\S\n]+(?P.+?)\', - 'updated': r'(changed):[^\S\n]+(?P.+?)\', - 'source': r'(source):[^\S\n]+(?P.+?)\', + 'description': r'(descr):[^\S\n]+(?P.+?)\n', + 'maintainer': r'(mnt-by):[^\S\n]+(?P.+?)\n', + 'updated': r'(changed):[^\S\n]+(?P.+?)\n', + 'source': r'(source):[^\S\n]+(?P.+?)\<', } }, } @@ -796,6 +796,8 @@ class ASNOrigin: asn=asn, retry_count=retry_count ) + break + except (WhoisLookupError, WhoisRateLimitError) as e: log.debug('ASN origin WHOIS lookup failed: {0}' @@ -809,17 +811,22 @@ class ASNOrigin: log.debug('Response not given, perform ASN origin ' 'HTTP lookup for: {0}'.format(asn)) - tmp = ASN_ORIGIN_HTTP['radb']['form_data'] - tmp[str(ASN_ORIGIN_HTTP['radb']['form_data_asn_field'] - )] = asn + # tmp = ASN_ORIGIN_HTTP['radb']['form_data'] + # tmp[str( + # ASN_ORIGIN_HTTP['radb']['form_data_asn_field'] + # )] = asn response = self._net.get_http_raw( - url=ASN_ORIGIN_HTTP['radb']['url'], + url=('{0}?advanced_query=1&keywords={1}&-T+option' + '=&ip_option=&-i=1&-i+option=origin' + ).format(ASN_ORIGIN_HTTP['radb']['url'], asn), retry_count=retry_count, - request_type='POST', - form_data=tmp + request_type='GET', + # form_data=tmp ) is_http = True # pragma: no cover + break + except HTTPLookupError as e: log.debug('ASN origin HTTP lookup failed: {0}' diff --git a/ipwhois/tests/online/test_asn.py b/ipwhois/tests/online/test_asn.py index 62cd804..76bc8ce 100644 --- a/ipwhois/tests/online/test_asn.py +++ b/ipwhois/tests/online/test_asn.py @@ -96,4 +96,5 @@ class TestASNOrigin(TestCommon): net = Net(address='74.125.225.229') asnorigin = ASNOrigin(net) - asnorigin.lookup(asn='15169', asn_methods=['whois', 'http']) + asnorigin.lookup(asn='15169', asn_methods=['whois']) + asnorigin.lookup(asn='15169', asn_methods=['http']) From ba1bebdf687246ba5abb1d28562457efa024d1b3 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 6 Aug 2019 18:27:56 -0500 Subject: [PATCH 11/36] Fixed bug in KRNIC queries due to a change in their service (#243) --- CHANGES.rst | 1 + ipwhois/nir.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 663cb00..f0998b7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,6 +15,7 @@ Changelog - Removed deprecated allow_permutations parameter (#230) - Fixed ASNOrigin lookups (#216) - Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216) +- Fixed bug in KRNIC queries due to a change in their service (#243) 1.1.0 (2019-02-01) ------------------ diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 2c470ac..ea53987 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -108,9 +108,14 @@ NIR_WHOIS = { }, 'krnic': { 'country_code': 'KR', - 'url': 'https://whois.kisa.or.kr/eng/whois.jsc', + 'url': 'https://xn--c79as89aj0e29b77z.xn--3e0b707e/eng/whois.jsc', 'request_type': 'POST', - 'request_headers': {'Accept': 'text/html'}, + 'request_headers': { + 'Accept': 'text/html', + 'Referer': ( + 'https://xn--c79as89aj0e29b77z.xn--3e0b707e/eng/whois.jsp' + ), + }, 'form_data_ip_field': 'query', 'fields': { 'name': r'(Organization Name)[\s]+\:[^\S\n]+(?P.+?)\n', From 05708cda9b235ca374f59cdbb3730bd8f8cb61c2 Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Mon, 10 Aug 2020 15:29:57 +0100 Subject: [PATCH 12/36] Fix issue where bulk_lookup_rdap() returns only one item result Add rdap lookups to the returned dictionary results --- ipwhois/experimental.py | 10 +++++----- ipwhois/rdap.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index a048bdf..85fff9b 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -253,15 +253,15 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, try: - results = ipasn.parse_fields_whois(asn_result) + asn_parsed = ipasn.parse_fields_whois(asn_result) except ASNRegistryError: # pragma: no cover continue # Add valid IP ASN result to asn_parsed_results for RDAP lookup - asn_parsed_results[ip] = results - stats[results['asn_registry']]['total'] += 1 + asn_parsed_results[ip] = asn_parsed + stats[asn_parsed['asn_registry']]['total'] += 1 # Set the list of IPs that are not allocated/failed ASN lookup stats['unallocated_addresses'] = list(k for k in addresses if k not in @@ -362,7 +362,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Perform the RDAP lookup. retry_count is set to 0 # here since we handle that in this function - results = rdap.lookup( + rdap_result = rdap.lookup( inc_raw=inc_raw, retry_count=0, asn_data=asn_data, depth=depth, excluded_entities=excluded_entities ) @@ -373,7 +373,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Lookup was successful, add to result. Set the nir # key to None as this is not supported # (yet - requires more queries) - results[ip] = results + results[ip] = rdap_result results[ip]['nir'] = None # Remove the IP from the lookup queue diff --git a/ipwhois/rdap.py b/ipwhois/rdap.py index 9bccbe1..d019f6d 100644 --- a/ipwhois/rdap.py +++ b/ipwhois/rdap.py @@ -553,7 +553,7 @@ class _RDAPNetwork(_RDAPCommon): self.vars[v] = self.json[v].strip() - except (KeyError, ValueError): + except (KeyError, ValueError, AttributeError): pass From b258ebd4c0073f2787e4defbb5fb3afce99aa58a Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Tue, 11 Aug 2020 09:51:58 +0100 Subject: [PATCH 13/36] Add test --- ipwhois/tests/online/test_experimental.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/online/test_experimental.py b/ipwhois/tests/online/test_experimental.py index 31ad57e..fc5042c 100644 --- a/ipwhois/tests/online/test_experimental.py +++ b/ipwhois/tests/online/test_experimental.py @@ -67,8 +67,22 @@ class TestExperimental(TestCommon): '115.1.2.3' # KRNIC ] + expected_stats = {'ip_input_total': 12, 'ip_unique_total': 12, 'ip_lookup_total': 12, + 'lacnic': {'failed': [], 'rate_limited': [], 'total': 2}, + 'ripencc': {'failed': [], 'rate_limited': [], 'total': 2}, + 'apnic': {'failed': [], 'rate_limited': [], 'total': 4}, + 'afrinic': {'failed': [], 'rate_limited': [], 'total': 2}, + 'arin': {'failed': [], 'rate_limited': [], 'total': 2}, + 'unallocated_addresses': []} + try: - self.assertIsInstance(bulk_lookup_rdap(addresses=ips), tuple) + result = bulk_lookup_rdap(addresses=ips) + self.assertIsInstance(result, tuple) + + results, stats = result + self.assertEqual(stats, expected_stats) + self.assertEqual(len(results), 12) + except ASNLookupError: pass except AssertionError as e: From a3253af24a3d67cafd2a0d90dfa01c239dee4142 Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Tue, 11 Aug 2020 23:28:09 +0100 Subject: [PATCH 14/36] Add asn data to rdap results --- ipwhois/experimental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 85fff9b..1bd989d 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -373,7 +373,8 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Lookup was successful, add to result. Set the nir # key to None as this is not supported # (yet - requires more queries) - results[ip] = rdap_result + results[ip]['asn'] = asn_data + results[ip]['rdap'] = rdap_result results[ip]['nir'] = None # Remove the IP from the lookup queue From fa9330ba4a645c880761becdefc983be6e364f9d Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Tue, 11 Aug 2020 23:58:50 +0100 Subject: [PATCH 15/36] Add asn data at the parent level --- ipwhois/experimental.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 1bd989d..33ee546 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -373,7 +373,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Lookup was successful, add to result. Set the nir # key to None as this is not supported # (yet - requires more queries) - results[ip]['asn'] = asn_data + results[ip] = asn_data results[ip]['rdap'] = rdap_result results[ip]['nir'] = None From fe7dd4fde02c170b22bbf08274677070b71d9659 Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Wed, 12 Aug 2020 09:22:39 +0100 Subject: [PATCH 16/36] merge asn and rdap data into the results --- ipwhois/experimental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 33ee546..9d597e9 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -374,7 +374,8 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # key to None as this is not supported # (yet - requires more queries) results[ip] = asn_data - results[ip]['rdap'] = rdap_result + results[ip].update(rdap_result) + results[ip]['nir'] = None # Remove the IP from the lookup queue From 40c69ccc3bda2dab73180eadb95e1266c37f66d6 Mon Sep 17 00:00:00 2001 From: Philip Hane Date: Wed, 2 Sep 2020 14:57:59 -0500 Subject: [PATCH 17/36] Update changes for #262 --- CHANGES.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f0998b7..e9c2319 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,8 @@ Changelog - Fixed ASNOrigin lookups (#216) - Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216) - Fixed bug in KRNIC queries due to a change in their service (#243) +- Fixed bug in experimental.bulk_lookup_rdap where only the last + result was returned (#262 - ameidatou) 1.1.0 (2019-02-01) ------------------ @@ -233,4 +235,4 @@ Changelog - Added support for IPv4Address or IPv6Address as the address arg in IPWhois. - Fixed file open encoding bug. Moved from open to io.open. - Fixed parameter in IPWhois ip defined checks. -- Fixed TestIPWhois.test_ip_invalid() assertions. \ No newline at end of file +- Fixed TestIPWhois.test_ip_invalid() assertions. From 29479698de478bf49039710ab4193cbdf33e7def Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Fri, 11 Sep 2020 14:08:22 +0000 Subject: [PATCH 18/36] Fix deprecation warnings due to invalid escape sequences. --- ipwhois/nir.py | 10 +++++----- ipwhois/utils.py | 36 ++++++++++++++++++------------------ ipwhois/whois.py | 18 +++++++++--------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/ipwhois/nir.py b/ipwhois/nir.py index ea53987..3d2f603 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -87,9 +87,9 @@ NIR_WHOIS = { 'updated': r'(\[Last Update\])[^\S\n]+(?P.*?)\n', 'nameservers': r'(\[Nameserver\])[^\S\n]+(?P.*?)\n', 'contact_admin': r'(\[Administrative Contact\])[^\S\n]+.+?\>' - '(?P.+?)\<\/A\>\n', + '(?P.+?)\\<\\/A\\>\n', 'contact_tech': r'(\[Technical Contact\])[^\S\n]+.+?\>' - '(?P.+?)\<\/A\>\n' + '(?P.+?)\\<\\/A\\>\n' }, 'contact_fields': { 'name': r'(\[Last, First\])[^\S\n]+(?P.*?)\n', @@ -125,9 +125,9 @@ NIR_WHOIS = { 'postal_code': r'(Zip Code)[\s]+\:[^\S\n]+(?P.+?)\n', 'created': r'(Registration Date)[\s]+\:[^\S\n]+(?P.+?)\n', 'contact_admin': r'(id="eng_isp_contact").+?\>(?P.*?)\<' - '\/div\>\n', + '\\/div\\>\n', 'contact_tech': r'(id="eng_user_contact").+?\>(?P.*?)\<' - '\/div\>\n' + '\\/div\\>\n' }, 'contact_fields': { 'name': r'(Name)[^\S\n]+?:[^\S\n]+?(?P.*?)\n', @@ -379,7 +379,7 @@ class NIRWhois: # and the start and end positions. for match in re.finditer( r'^(IPv4 Address)[\s]+:[^\S\n]+((.+?)[^\S\n]-[^\S\n](.+?)' - '[^\S\n]\((.+?)\)|.+)$', + '[^\\S\n]\\((.+?)\\)|.+)$', response, re.MULTILINE ): diff --git a/ipwhois/utils.py b/ipwhois/utils.py index 1df4a91..93297fd 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -87,29 +87,29 @@ IETF_RFC_REFERENCES = { IP_REGEX = ( r'(?P' # IPv4 - '(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' + r'(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' # IPv6 - '|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' - '{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' - '2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]' - '{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d' - '\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|' - '((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' - '2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]' - '{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)' - '(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' - '(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' - '\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' + r'|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' + r'{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' + r'2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]' + r'{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d' + r'\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|' + r'((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' + r'2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]' + r'{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)' + r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' + r'(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' + r'\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' '[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' - '{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' - '?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' - '25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' - ')|:)))(%.+)?))\]?' + r'{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' + r'?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' + r'25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' + r')|:)))(%.+)?))\]?' # Optional IPv4 Port - '((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' + r'((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' # Optional CIDR block - '))|(\/(?:[012]\d?|3[012]?|[4-9])))?' + r'))|(\/(?:[012]\d?|3[012]?|[4-9])))?' ')' ) diff --git a/ipwhois/whois.py b/ipwhois/whois.py index abc3361..99dc07c 100644 --- a/ipwhois/whois.py +++ b/ipwhois/whois.py @@ -67,7 +67,7 @@ RIR_WHOIS = { 'name': r'(NetName):[^\S\n]+(?P.+?)\n', 'handle': r'(NetHandle):[^\S\n]+(?P.+?)\n', 'description': r'(OrgName|CustName):[^\S\n]+(?P.+?)' - '(?=(\n\S):?)', + '(?=(\n\\S):?)', 'country': r'(Country):[^\S\n]+(?P.+?)\n', 'state': r'(StateProv):[^\S\n]+(?P.+?)\n', 'city': r'(City):[^\S\n]+(?P.+?)\n', @@ -75,7 +75,7 @@ RIR_WHOIS = { 'postal_code': r'(PostalCode):[^\S\n]+(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(RegDate):[^\S\n]+(?P.+?)\n', 'updated': r'(Updated):[^\S\n]+(?P.+?)\n', @@ -92,7 +92,7 @@ RIR_WHOIS = { 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': ( r'(created):[^\S\n]+(?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]' @@ -115,7 +115,7 @@ RIR_WHOIS = { 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'updated': r'(changed):[^\S\n]+.*(?P[0-9]{8}).*?\n' }, @@ -129,7 +129,7 @@ RIR_WHOIS = { 'country': r'(country):[^\S\n]+(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(created):[^\S\n]+(?P[0-9]{8}).*?\n', 'updated': r'(changed):[^\S\n]+(?P[0-9]{8}).*?\n' @@ -146,7 +146,7 @@ RIR_WHOIS = { 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), } } @@ -166,7 +166,7 @@ RWHOIS = { 'postal_code': r'(network:Postal-Code):(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(network:Created):(?P.+?)\n', 'updated': r'(network:Updated):(?P.+?)\n' @@ -454,7 +454,7 @@ class Whois: for addr in net_range.split(', '): count = addr.count('.') - if count is not 0 and count < 4: + if count != 0 and count < 4: addr_split = addr.strip().split('/') for i in range(count + 1, 4): @@ -627,7 +627,7 @@ class Whois: # Only fetch the response if we haven't already. if response is None or (not is_offline and - asn_data['asn_registry'] is not 'arin'): + asn_data['asn_registry'] != 'arin'): log.debug('Response not given, perform WHOIS lookup for {0}' .format(self._net.address_str)) From 0516872ab79356774d5729dec8b3d61ee287717e Mon Sep 17 00:00:00 2001 From: Philip Hane Date: Mon, 14 Sep 2020 11:45:23 -0500 Subject: [PATCH 19/36] Fixed deprecation warnings due to invalid escape sequences --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e9c2319..1c2a7e2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,8 @@ Changelog - Fixed bug in KRNIC queries due to a change in their service (#243) - Fixed bug in experimental.bulk_lookup_rdap where only the last result was returned (#262 - ameidatou) +- Fixed deprecation warnings due to invalid escape sequences + (#272 - tirkarthi) 1.1.0 (2019-02-01) ------------------ From 00879c931286ce87f3225a426437fc5f8e9b0554 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 12:30:10 -0500 Subject: [PATCH 20/36] Added support for Python 3.8 (#267) --- .travis.yml | 1 + CHANGES.rst | 1 + UPGRADING.rst | 1 + setup.py | 1 + 4 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index a363b5e..4f08fb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 3.5 - 3.6 - 3.7 + - 3.8 install: - pip install --upgrade setuptools - pip install --upgrade pip diff --git a/CHANGES.rst b/CHANGES.rst index 1c2a7e2..7d17b5c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,7 @@ Changelog result was returned (#262 - ameidatou) - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) +- Added support for Python 3.8 (#267) 1.1.0 (2019-02-01) ------------------ diff --git a/UPGRADING.rst b/UPGRADING.rst index d26f5f2..181bb87 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -22,6 +22,7 @@ v1.2.0 nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact - Removed deprecated asn_alts parameter - Removed deprecated allow_permutations parameter +- Added support for Python 3.8 ****** v1.1.0 diff --git a/setup.py b/setup.py index 9103dec..949dee5 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Internet', 'Topic :: Software Development', ] From 67b9c89a12a9ae9f082c2bf7261cbe5db566a95d Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 12:55:25 -0500 Subject: [PATCH 21/36] 38 regex fix attempt (#267) --- ipwhois/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/utils.py b/ipwhois/utils.py index 93297fd..aad51b8 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -88,7 +88,7 @@ IP_REGEX = ( r'(?P' # IPv4 r'(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' - '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' + r'(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' # IPv6 r'|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' r'{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' @@ -110,7 +110,7 @@ IP_REGEX = ( r'((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' # Optional CIDR block r'))|(\/(?:[012]\d?|3[012]?|[4-9])))?' - ')' + r')' ) From 5bdc0dcb65e1d9b39e478a6594be4f7d5ff58ba3 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 13:18:07 -0500 Subject: [PATCH 22/36] 38 fix tests (#267) --- ipwhois/tests/test_utils.py | 110 ++++++++++++++++++++++++++---------- ipwhois/utils.py | 6 +- 2 files changed, 86 insertions(+), 30 deletions(-) diff --git a/ipwhois/tests/test_utils.py b/ipwhois/tests/test_utils.py index 7a26950..44b3fc3 100644 --- a/ipwhois/tests/test_utils.py +++ b/ipwhois/tests/test_utils.py @@ -64,35 +64,35 @@ class TestFunctions(TestCommon): self.assertRaises(ValueError, ipv4_is_defined, '192.168.0.256') self.assertRaises(AddressValueError, ipv4_is_defined, 1234) - self.assertEquals(ipv4_is_defined('74.125.225.229'), (False, '', '')) + self.assertEqual(ipv4_is_defined('74.125.225.229'), (False, '', '')) - self.assertEquals(ipv4_is_defined('0.0.0.0'), + self.assertEqual(ipv4_is_defined('0.0.0.0'), (True, 'This Network', 'RFC 1122, Section 3.2.1.3')) - self.assertEquals(ipv4_is_defined('127.0.0.0'), + self.assertEqual(ipv4_is_defined('127.0.0.0'), (True, 'Loopback', 'RFC 1122, Section 3.2.1.3')) - self.assertEquals(ipv4_is_defined('169.254.0.0'), + self.assertEqual(ipv4_is_defined('169.254.0.0'), (True, 'Link Local', 'RFC 3927')) - self.assertEquals(ipv4_is_defined('192.0.0.0'), + self.assertEqual(ipv4_is_defined('192.0.0.0'), (True, 'IETF Protocol Assignments', 'RFC 5736')) - self.assertEquals(ipv4_is_defined('192.0.2.0'), + self.assertEqual(ipv4_is_defined('192.0.2.0'), (True, 'TEST-NET-1', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('192.88.99.0'), + self.assertEqual(ipv4_is_defined('192.88.99.0'), (True, '6to4 Relay Anycast', 'RFC 3068')) - self.assertEquals(ipv4_is_defined('198.18.0.0'), + self.assertEqual(ipv4_is_defined('198.18.0.0'), (True, 'Network Interconnect Device Benchmark Testing', 'RFC 2544')) - self.assertEquals(ipv4_is_defined('198.51.100.0'), + self.assertEqual(ipv4_is_defined('198.51.100.0'), (True, 'TEST-NET-2', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('203.0.113.0'), + self.assertEqual(ipv4_is_defined('203.0.113.0'), (True, 'TEST-NET-3', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('224.0.0.0'), + self.assertEqual(ipv4_is_defined('224.0.0.0'), (True, 'Multicast', 'RFC 3171')) - self.assertEquals(ipv4_is_defined('255.255.255.255'), + self.assertEqual(ipv4_is_defined('255.255.255.255'), (True, 'Limited Broadcast', 'RFC 919, Section 7')) - self.assertEquals(ipv4_is_defined('192.168.0.1'), + self.assertEqual(ipv4_is_defined('192.168.0.1'), (True, 'Private-Use Networks', 'RFC 1918')) - self.assertEquals(ipv4_is_defined('198.97.38.0'), + self.assertEqual(ipv4_is_defined('198.97.38.0'), (True, 'IANA Reserved', '')) def test_ipv6_is_defined(self): @@ -105,31 +105,31 @@ class TestFunctions(TestCommon): '2001:4860:4860::8888::1234') self.assertRaises(AddressValueError, ipv6_is_defined, 1234) - self.assertEquals(ipv6_is_defined('2001:4860:4860::8888'), + self.assertEqual(ipv6_is_defined('2001:4860:4860::8888'), (False, '', '')) - self.assertEquals(ipv6_is_defined('ff00::'), + self.assertEqual(ipv6_is_defined('ff00::'), (True, 'Multicast', 'RFC 4291, Section 2.7')) - self.assertEquals(ipv6_is_defined('0:0:0:0:0:0:0:0'), + self.assertEqual(ipv6_is_defined('0:0:0:0:0:0:0:0'), (True, 'Unspecified', 'RFC 4291, Section 2.5.2')) - self.assertEquals(ipv6_is_defined('0:0:0:0:0:0:0:1'), + self.assertEqual(ipv6_is_defined('0:0:0:0:0:0:0:1'), (True, 'Loopback', 'RFC 4291, Section 2.5.3')) - self.assertEquals(ipv6_is_defined('100::'), + self.assertEqual(ipv6_is_defined('100::'), (True, 'Reserved', 'RFC 4291')) - self.assertEquals(ipv6_is_defined('fe80::'), + self.assertEqual(ipv6_is_defined('fe80::'), (True, 'Link-Local', 'RFC 4291, Section 2.5.6')) - self.assertEquals(ipv6_is_defined('fec0::'), + self.assertEqual(ipv6_is_defined('fec0::'), (True, 'Site-Local', 'RFC 4291, Section 2.5.7')) - self.assertEquals(ipv6_is_defined('fc00::'), + self.assertEqual(ipv6_is_defined('fc00::'), (True, 'Unique Local Unicast', 'RFC 4193')) def test_unique_everseen(self): input_list = ['b', 'a', 'c', 'a', 'b', 'x', 'a'] - self.assertEquals(list(unique_everseen(input_list)), + self.assertEqual(list(unique_everseen(input_list)), ['b', 'a', 'c', 'x']) - self.assertEquals(list(unique_everseen(input_list, str.lower)), + self.assertEqual(list(unique_everseen(input_list, str.lower)), ['b', 'a', 'c', 'x']) def test_unique_addresses(self): @@ -150,14 +150,14 @@ class TestFunctions(TestCommon): '2001:4860:4860::8888': {'count': 2, 'ports': {'443': 1}} } - self.assertEquals(unique_addresses(input_data), expected_result) + self.assertEqual(unique_addresses(input_data), expected_result) data_dir = path.dirname(__file__) fp = str(data_dir) + '/rdap.json' # Expected result is different on 2.x vs 3.x, possible issues with # ipaddr vs ipaddress output. Investigation pending... - if sys.version_info >= (3, 3): + if (3, 3) <= sys.version_info < (3, 8): fp_expected_result = { '74.125.225.0/24': {'count': 1, 'ports': {}}, @@ -203,7 +203,59 @@ class TestFunctions(TestCommon): '210.0.0.0/8': {'count': 1, 'ports': {}} } - self.assertEquals(unique_addresses(file_path=fp), + self.assertEqual(unique_addresses(file_path=fp), + fp_expected_result) + + if sys.version_info >= (3, 8): + + fp_expected_result = { + '196.0.0.0': {'count': 1, 'ports': {}}, + '196.11.239.0': {'count': 2, 'ports': {}}, + '196.11.240.0/23': {'count': 1, 'ports': {}}, + '196.11.240.215': {'count': 2, 'ports': {}}, + '196.11.246.255': {'count': 2, 'ports': {}}, + '196.255.255.255': {'count': 1, 'ports': {}}, + '200.57.128.0/20': {'count': 1, 'ports': {}}, + '200.57.141.161': {'count': 7, 'ports': {}}, + '2001:200::/23': {'count': 2, 'ports': {}}, + '2001:240:10c:1::ca20:9d1d': + {'count': 2, 'ports': {}}, + '2001:240::': {'count': 1, 'ports': {}}, + '2001:240::/32': {'count': 6, 'ports': {}}, + '2001:240:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '2001:4200::/23': {'count': 1, 'ports': {}}, + '2001:43f8:7b0::': {'count': 3, 'ports': {}}, + '2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '2001:4860:4860::8888': {'count': 10, 'ports': {}}, + '2001:4860::': {'count': 2, 'ports': {}}, + '2001:4860::/32': {'count': 1, 'ports': {}}, + '2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '210.0.0.0': {'count': 1, 'ports': {}}, + '210.0.0.0/8': {'count': 1, 'ports': {}}, + '210.107.0.0': {'count': 2, 'ports': {}}, + '210.107.0.0/17': {'count': 6, 'ports': {}}, + '210.107.127.255': {'count': 2, 'ports': {}}, + '210.107.73.73': {'count': 2, 'ports': {}}, + '210.255.255.255': {'count': 1, 'ports': {}}, + '2801:10:c000::': {'count': 7, 'ports': {}}, + '2a00:2380::/25': {'count': 1, 'ports': {}}, + '2a00:2381:ffff::1': {'count': 4, 'ports': {}}, + '62.239.0.0/16': {'count': 1, 'ports': {}}, + '62.239.237.0': {'count': 1, 'ports': {}}, + '62.239.237.0/32': {'count': 1, 'ports': {}}, + '62.239.237.1': {'count': 4, 'ports': {}}, + '62.239.237.255': {'count': 1, 'ports': {}}, + '62.239.237.255/32': {'count': 1, 'ports': {}}, + '74.125.0.0': {'count': 2, 'ports': {}}, + '74.125.225.0/24': {'count': 1, 'ports': {}}, + '74.125.225.229': {'count': 8, 'ports': {}}, + '74.125.255.255': {'count': 1, 'ports': {}} + } + + self.assertEqual(unique_addresses(file_path=fp), fp_expected_result) else: @@ -261,8 +313,8 @@ class TestFunctions(TestCommon): def test_ipv4_generate_random(self): - self.assertEquals(len(list(ipv4_generate_random(1000))), 1000) + self.assertEqual(len(list(ipv4_generate_random(1000))), 1000) def test_ipv6_generate_random(self): - self.assertEquals(len(list(ipv6_generate_random(1000))), 1000) + self.assertEqual(len(list(ipv6_generate_random(1000))), 1000) diff --git a/ipwhois/utils.py b/ipwhois/utils.py index aad51b8..920ed0f 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -101,7 +101,7 @@ IP_REGEX = ( r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' r'(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' r'\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' - '[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' + r'[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' r'{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' r'?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' r'25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' @@ -212,6 +212,7 @@ def get_countries(is_legacy_xml=False): # Read the file. data = f.read() + f.close() # Check if there is data. if not data: # pragma: no cover @@ -258,6 +259,8 @@ def get_countries(is_legacy_xml=False): # Add to the countries dictionary. countries[code] = name + f.close() + return countries @@ -506,6 +509,7 @@ def unique_addresses(data=None, file_path=None): # Read the file. file_data = f.read() + f.close() pattern = re.compile( str(IP_REGEX), From e3d85e97c2d6cb5d84d2a3c1df667c1a161010c1 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 13:31:42 -0500 Subject: [PATCH 23/36] 38 fix tests 2 (#267) --- ipwhois/tests/test_utils.py | 53 ++----------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/ipwhois/tests/test_utils.py b/ipwhois/tests/test_utils.py index 44b3fc3..fdbd4d1 100644 --- a/ipwhois/tests/test_utils.py +++ b/ipwhois/tests/test_utils.py @@ -157,56 +157,7 @@ class TestFunctions(TestCommon): # Expected result is different on 2.x vs 3.x, possible issues with # ipaddr vs ipaddress output. Investigation pending... - if (3, 3) <= sys.version_info < (3, 8): - - fp_expected_result = { - '74.125.225.0/24': {'count': 1, 'ports': {}}, - '62.239.0.0/16': {'count': 1, 'ports': {}}, - '2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': - {'count': 1, 'ports': {}}, - '210.0.0.0': {'count': 1, 'ports': {}}, - '196.11.240.0/23': {'count': 1, 'ports': {}}, - '2001:240:10c:1::ca20:9d1d': {'count': 2, 'ports': {}}, - '196.11.240.215': {'count': 2, 'ports': {}}, - '62.239.237.0/32': {'count': 1, 'ports': {}}, - '210.107.0.0/17': {'count': 6, 'ports': {}}, - '2001:4860::/32': {'count': 1, 'ports': {}}, - '210.107.73.73': {'count': 2, 'ports': {}}, - '210.107.0.0': {'count': 2, 'ports': {}}, - '2001:200::/23': {'count': 2, 'ports': {}}, - '2001:240:ffff:ffff:ffff:ffff:ffff:ffff': - {'count': 1, 'ports': {}}, - '210.255.255.255': {'count': 1, 'ports': {}}, - '2001:43f8:7b0::': {'count': 3, 'ports': {}}, - '196.255.255.255': {'count': 1, 'ports': {}}, - '2001:240::/32': {'count': 6, 'ports': {}}, - '196.0.0.0': {'count': 1, 'ports': {}}, - '2001:240::': {'count': 1, 'ports': {}}, - '196.11.246.255': {'count': 2, 'ports': {}}, - '196.11.239.0': {'count': 2, 'ports': {}}, - '2001:4200::/23': {'count': 1, 'ports': {}}, - '2a00:2380::/25': {'count': 1, 'ports': {}}, - '200.57.128.0/20': {'count': 1, 'ports': {}}, - '62.239.237.255': {'count': 1, 'ports': {}}, - '2001:4860:4860::8888': {'count': 10, 'ports': {}}, - '2001:4860::': {'count': 2, 'ports': {}}, - '2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': - {'count': 1, 'ports': {}}, - '74.125.225.229': {'count': 8, 'ports': {}}, - '210.107.127.255': {'count': 2, 'ports': {}}, - '200.57.141.161': {'count': 7, 'ports': {}}, - '62.239.237.255/32': {'count': 1, 'ports': {}}, - '2801:10:c000::': {'count': 7, 'ports': {}}, - '2a00:2381:ffff::1': {'count': 4, 'ports': {}}, - '62.239.237.0': {'count': 1, 'ports': {}}, - '62.239.237.1': {'count': 4, 'ports': {}}, - '210.0.0.0/8': {'count': 1, 'ports': {}} - } - - self.assertEqual(unique_addresses(file_path=fp), - fp_expected_result) - - if sys.version_info >= (3, 8): + if sys.version_info >= (3, 3): fp_expected_result = { '196.0.0.0': {'count': 1, 'ports': {}}, @@ -256,7 +207,7 @@ class TestFunctions(TestCommon): } self.assertEqual(unique_addresses(file_path=fp), - fp_expected_result) + fp_expected_result) else: From 17039b8c3b6f2c6b5fcc2e86a768f63f2f3b519e Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 13:44:34 -0500 Subject: [PATCH 24/36] 38 fix tests 3 (#267) --- ipwhois/tests/test_utils.py | 51 ++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/test_utils.py b/ipwhois/tests/test_utils.py index fdbd4d1..9dc041f 100644 --- a/ipwhois/tests/test_utils.py +++ b/ipwhois/tests/test_utils.py @@ -157,7 +157,56 @@ class TestFunctions(TestCommon): # Expected result is different on 2.x vs 3.x, possible issues with # ipaddr vs ipaddress output. Investigation pending... - if sys.version_info >= (3, 3): + if (3, 3) <= sys.version_info < (3, 8): + + fp_expected_result = { + '74.125.225.0/24': {'count': 1, 'ports': {}}, + '62.239.0.0/16': {'count': 1, 'ports': {}}, + '2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '210.0.0.0': {'count': 1, 'ports': {}}, + '196.11.240.0/23': {'count': 1, 'ports': {}}, + '2001:240:10c:1::ca20:9d1d': {'count': 2, 'ports': {}}, + '196.11.240.215': {'count': 2, 'ports': {}}, + '62.239.237.0/32': {'count': 1, 'ports': {}}, + '210.107.0.0/17': {'count': 6, 'ports': {}}, + '2001:4860::/32': {'count': 1, 'ports': {}}, + '210.107.73.73': {'count': 2, 'ports': {}}, + '210.107.0.0': {'count': 2, 'ports': {}}, + '2001:200::/23': {'count': 2, 'ports': {}}, + '2001:240:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '210.255.255.255': {'count': 1, 'ports': {}}, + '2001:43f8:7b0::': {'count': 3, 'ports': {}}, + '196.255.255.255': {'count': 1, 'ports': {}}, + '2001:240::/32': {'count': 6, 'ports': {}}, + '196.0.0.0': {'count': 1, 'ports': {}}, + '2001:240::': {'count': 1, 'ports': {}}, + '196.11.246.255': {'count': 2, 'ports': {}}, + '196.11.239.0': {'count': 2, 'ports': {}}, + '2001:4200::/23': {'count': 1, 'ports': {}}, + '2a00:2380::/25': {'count': 1, 'ports': {}}, + '200.57.128.0/20': {'count': 1, 'ports': {}}, + '62.239.237.255': {'count': 1, 'ports': {}}, + '2001:4860:4860::8888': {'count': 10, 'ports': {}}, + '2001:4860::': {'count': 2, 'ports': {}}, + '2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '74.125.225.229': {'count': 8, 'ports': {}}, + '210.107.127.255': {'count': 2, 'ports': {}}, + '200.57.141.161': {'count': 7, 'ports': {}}, + '62.239.237.255/32': {'count': 1, 'ports': {}}, + '2801:10:c000::': {'count': 7, 'ports': {}}, + '2a00:2381:ffff::1': {'count': 4, 'ports': {}}, + '62.239.237.0': {'count': 1, 'ports': {}}, + '62.239.237.1': {'count': 4, 'ports': {}}, + '210.0.0.0/8': {'count': 1, 'ports': {}} + } + + self.assertEqual(unique_addresses(file_path=fp), + fp_expected_result) + + elif sys.version_info >= (3, 8): fp_expected_result = { '196.0.0.0': {'count': 1, 'ports': {}}, From 512efd3a5c99ab97f4ce2313208e30453cff4be3 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 14:29:33 -0500 Subject: [PATCH 25/36] Fixed travis build warnings (#268) --- .travis.yml | 2 +- CHANGES.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f08fb5..845f994 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -sudo: required +os: linux dist: xenial python: - 2.7 diff --git a/CHANGES.rst b/CHANGES.rst index 7d17b5c..0bf8a51 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,6 +21,7 @@ Changelog - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) - Added support for Python 3.8 (#267) +- Fixed travis build warnings (#268) 1.1.0 (2019-02-01) ------------------ From 70981aea8660a1362de3e9efd251a0aa90c9dc11 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 15:45:54 -0500 Subject: [PATCH 26/36] Pinned requirements (#274) --- CHANGES.rst | 1 + UPGRADING.rst | 1 + ipwhois/docs/requirements.txt | 2 +- requirements/python2.txt | 4 ++-- requirements/python3.txt | 2 +- setup.py | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0bf8a51..aa410a7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,7 @@ Changelog (#272 - tirkarthi) - Added support for Python 3.8 (#267) - Fixed travis build warnings (#268) +- Pinned requirements (#274) 1.1.0 (2019-02-01) ------------------ diff --git a/UPGRADING.rst b/UPGRADING.rst index 181bb87..d689a4f 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -23,6 +23,7 @@ v1.2.0 - Removed deprecated asn_alts parameter - Removed deprecated allow_permutations parameter - Added support for Python 3.8 +- Pinned requirements ****** v1.1.0 diff --git a/ipwhois/docs/requirements.txt b/ipwhois/docs/requirements.txt index b7845ba..9a90c11 100644 --- a/ipwhois/docs/requirements.txt +++ b/ipwhois/docs/requirements.txt @@ -1,3 +1,3 @@ sphinx sphinx_rtd_theme -dnspython +dnspython<=2.0.0 diff --git a/requirements/python2.txt b/requirements/python2.txt index b7f70e7..6f6db6b 100644 --- a/requirements/python2.txt +++ b/requirements/python2.txt @@ -1,2 +1,2 @@ -dnspython -ipaddr +dnspython<=2.0.0 +ipaddr==2.2.0 diff --git a/requirements/python3.txt b/requirements/python3.txt index 2f73596..337b63d 100644 --- a/requirements/python3.txt +++ b/requirements/python3.txt @@ -1 +1 @@ -dnspython +dnspython<=2.0.0 diff --git a/setup.py b/setup.py index 949dee5..e651eaf 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ PACKAGES = ['ipwhois'] PACKAGE_DATA = {'ipwhois': ['data/*.xml', 'data/*.csv']} -INSTALL_REQUIRES = ['dnspython', 'ipaddr;python_version<"3.3"'] +INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"'] setup( name=NAME, From 1f5ddcf01293985956b9e86f06a07b7e7105b53e Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 11:48:12 -0500 Subject: [PATCH 27/36] Added ip_failed_total to bulk_lookup_rdap (#235) --- CHANGES.rst | 2 ++ EXPERIMENTAL.rst | 26 ++++++++++++++------------ ipwhois/experimental.py | 7 ++++++- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index aa410a7..b6beac5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -23,6 +23,8 @@ Changelog - Added support for Python 3.8 (#267) - Fixed travis build warnings (#268) - Pinned requirements (#274) +- Added ip_failed_total key to stats dictionary in + experimental.bulk_lookup_rdap (#235) 1.1.0 (2019-02-01) ------------------ diff --git a/EXPERIMENTAL.rst b/EXPERIMENTAL.rst index d95f635..0c6179f 100644 --- a/EXPERIMENTAL.rst +++ b/EXPERIMENTAL.rst @@ -68,21 +68,21 @@ Basic usage >>>> pprint(results.split('\n')) [ - "Bulk mode; whois.cymru.com [2017-07-30 23:02:21 +0000]", - "15169 | 74.125.225.229 | 74.125.225.0/24 | US | arin | 2007-03-13 | GOOGLE - Google Inc., US", - "15169 | 2001:4860:4860::8888 | 2001:4860::/32 | US | arin | 2005-03-14 | GOOGLE - Google Inc., US", + "Bulk mode; whois.cymru.com [2020-09-15 16:42:29 +0000]", + "15169 | 74.125.225.229 | 74.125.225.0/24 | US | arin | 2007-03-13 | GOOGLE, US", + "15169 | 2001:4860:4860::8888 | 2001:4860::/32 | US | arin | 2005-03-14 | GOOGLE, US", "2856 | 62.239.237.1 | 62.239.0.0/16 | GB | ripencc | 2001-01-02 | BT-UK-AS BTnet UK Regional network, GB", "2856 | 2a00:2381:ffff::1 | 2a00:2380::/25 | GB | ripencc | 2007-08-29 | BT-UK-AS BTnet UK Regional network, GB", - "3786 | 210.107.73.73 | 210.107.0.0/17 | KR | apnic | | LGDACOM LG DACOM Corporation, KR", + "3786 | 210.107.73.73 | 210.107.0.0/17 | KR | apnic | 1997-08-29 | LGDACOM LG DACOM Corporation, KR", "2497 | 2001:240:10c:1::ca20:9d1d | 2001:240::/32 | JP | apnic | 2000-03-08 | IIJ Internet Initiative Japan Inc., JP", "19373 | 200.57.141.161 | 200.57.128.0/20 | MX | lacnic | 2000-12-04 | Triara.com, S.A. de C.V., MX", "NA | 2801:10:c000:: | NA | CO | lacnic | 2013-10-29 | NA", - "12091 | 196.11.240.215 | 196.11.240.0/24 | ZA | afrinic | | MTNNS-1, ZA", + "12091 | 196.11.240.215 | 196.11.240.0/24 | ZA | afrinic | 1994-07-21 | MTNNS-1, ZA", "37578 | 2001:43f8:7b0:: | 2001:43f8:7b0::/48 | KE | afrinic | 2013-03-22 | Tespok, KE", - "4730 | 133.1.2.5 | 133.1.0.0/16 | JP | apnic | | ODINS Osaka University, JP", - "4134 | 115.1.2.3 | 115.0.0.0/14 | KR | apnic | 2008-07-01 | CHINANET-BACKBONE No.31,Jin-rong Street, CN", + "4730 | 133.1.2.5 | 133.1.0.0/16 | JP | apnic | 1997-03-01 | ODINS Osaka University, JP", + "4766 | 115.1.2.3 | 115.0.0.0/12 | KR | apnic | 2008-07-01 | KIXS-AS-KR Korea Telecom, KR", "" - } + ] .. GET_BULK_ASN_WHOIS_OUTPUT_BASIC END @@ -175,11 +175,14 @@ The stats dictionary returned by ipwhois.experimental.bulk_lookup_rdap() 'ip_lookup_total' (int) - The total number of addresses that lookups were attempted for, excluding any that failed ASN registry checks. + 'ip_failed_total' (int) - The total number of addresses that + lookups failed for. Excludes any that failed initially, but + succeeded after further retries. 'lacnic' (dict) - { 'failed' (list) - The addresses that failed to lookup. Excludes any that failed initially, but succeeded after - futher retries. + further retries. 'rate_limited' (list) - The addresses that encountered rate-limiting. Unless an address is also in 'failed', it eventually succeeded. @@ -222,9 +225,7 @@ Basic usage "total": 2 }, "apnic": { - "failed": [ - "115.1.2.3" - ], + "failed": [], "rate_limited": [], "total": 4 }, @@ -233,6 +234,7 @@ Basic usage "rate_limited": [], "total": 2 }, + "ip_failed_total": 0, "ip_input_total": 12, "ip_lookup_total": 12, "ip_unique_total": 12, diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 9d597e9..8f6ce23 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -158,11 +158,14 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, 'ip_lookup_total' (int) - The total number of addresses that lookups were attempted for, excluding any that failed ASN registry checks. + 'ip_failed_total' (int) - The total number of addresses that + lookups failed for. Excludes any that failed initially, but + succeeded after further retries. 'lacnic' (dict) - { 'failed' (list) - The addresses that failed to lookup. Excludes any that failed initially, but succeeded after - futher retries. + further retries. 'rate_limited' (list) - The addresses that encountered rate-limiting. Unless an address is also in 'failed', it eventually succeeded. @@ -196,6 +199,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, 'ip_input_total': len(addresses), 'ip_unique_total': 0, 'ip_lookup_total': 0, + 'ip_failed_total': 0, 'lacnic': {'failed': [], 'rate_limited': [], 'total': 0}, 'ripencc': {'failed': [], 'rate_limited': [], 'total': 0}, 'apnic': {'failed': [], 'rate_limited': [], 'total': 0}, @@ -425,6 +429,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, del asn_parsed_results[ip] stats[rir]['failed'].append(ip) + stats['ip_failed_total'] += 1 if rir == 'lacnic': From c4e71b6dc17023e82b38a15e4aea9198df455cb1 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 11:58:13 -0500 Subject: [PATCH 28/36] Fixed test for ip_failed_total (#235) --- ipwhois/tests/online/test_experimental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/online/test_experimental.py b/ipwhois/tests/online/test_experimental.py index fc5042c..6ea9b47 100644 --- a/ipwhois/tests/online/test_experimental.py +++ b/ipwhois/tests/online/test_experimental.py @@ -67,7 +67,8 @@ class TestExperimental(TestCommon): '115.1.2.3' # KRNIC ] - expected_stats = {'ip_input_total': 12, 'ip_unique_total': 12, 'ip_lookup_total': 12, + expected_stats = {'ip_input_total': 12, 'ip_unique_total': 12, + 'ip_lookup_total': 12, 'ip_failed_total': 0, 'lacnic': {'failed': [], 'rate_limited': [], 'total': 2}, 'ripencc': {'failed': [], 'rate_limited': [], 'total': 2}, 'apnic': {'failed': [], 'rate_limited': [], 'total': 4}, From 3ffcdafe0dbdac60b7e116b506ee5b4580674e75 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 14:42:47 -0500 Subject: [PATCH 29/36] Fixed bug in root and sub-entities not getting queried/data (#247) --- CHANGES.rst | 4 + RDAP.rst | 13 +++ UPGRADING.rst | 4 + ipwhois/ipwhois.py | 8 +- ipwhois/rdap.py | 172 +++++++++++++++++++++++++------------ ipwhois/tests/test_rdap.py | 14 +-- 6 files changed, 154 insertions(+), 61 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b6beac5..2b25efe 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,10 @@ Changelog result was returned (#262 - ameidatou) - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) +- Fixed bug in root and sub-entities not getting queried/data (#247) +- Added new argument root_ent_check to IPWhois.lookup_rdap and + RDAP.lookup. Set this to False to revert to old functionality - missing data, + but less queries (#247) - Added support for Python 3.8 (#267) - Fixed travis build warnings (#268) - Pinned requirements (#274) diff --git a/RDAP.rst b/RDAP.rst index 9d32b50..0f8d0fa 100644 --- a/RDAP.rst +++ b/RDAP.rst @@ -73,6 +73,10 @@ Arguments supported by IPWhois.lookup_rdap(). | | | pulling ASN information via dns, in order to | | | | get the ASN description. Defaults to True. | +--------------------+--------+-----------------------------------------------+ +| root_ent_check | bool | If True, will perform additional RDAP HTTP | +| | | queries for missing entity data at the root | +| | | level. Defaults to True. | ++--------------------+--------+-----------------------------------------------+ .. _rdap-output: @@ -593,3 +597,12 @@ this very low for bulk queries, or disable completely by setting retry_count=0. Note that setting this result too low may cause a larger number of IP lookups to fail. + +root_ent_check +^^^^^^^^^^^^^^ + +When root level entities (depth=0) are missing vcard data, additional +entity specific HTTP lookups are performed. In the past, you would expect +depth=0 to mean a single lookup per IP. This was a bug and has been fixed as of +v1.2.0. Set this to False to revert back to the old method, although you will be +missing entity specific data. diff --git a/UPGRADING.rst b/UPGRADING.rst index d689a4f..300da87 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -22,6 +22,10 @@ v1.2.0 nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact - Removed deprecated asn_alts parameter - Removed deprecated allow_permutations parameter +- Added new argument root_ent_check to IPWhois.lookup_rdap and + RDAP.lookup. Set this to False to revert to old functionality - missing data, + but less queries. If you leave this set to default of True, you will notice + more queries and potentially more rate-limiting. - Added support for Python 3.8 - Pinned requirements diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index 0c3e179..d5368e8 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -199,7 +199,7 @@ class IPWhois: excluded_entities=None, bootstrap=False, rate_limit_timeout=120, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, - get_asn_description=True): + get_asn_description=True, root_ent_check=True): """ The function for retrieving and parsing whois information for an IP address via HTTP (RDAP). @@ -247,6 +247,9 @@ class IPWhois: get_asn_description (:obj:`bool`): Whether to run an additional query when pulling ASN information via dns, in order to get the ASN description. Defaults to True. + root_ent_check (:obj:`bool`): If True, will perform + additional RDAP HTTP queries for missing entity data at the + root level. Defaults to True. Returns: dict: The IP RDAP lookup results @@ -305,7 +308,8 @@ class IPWhois: inc_raw=inc_raw, retry_count=retry_count, asn_data=asn_data, depth=depth, excluded_entities=excluded_entities, response=response, bootstrap=bootstrap, - rate_limit_timeout=rate_limit_timeout + rate_limit_timeout=rate_limit_timeout, + root_ent_check=root_ent_check ) # Add the RDAP information to the return dictionary. diff --git a/ipwhois/rdap.py b/ipwhois/rdap.py index d019f6d..1e80338 100644 --- a/ipwhois/rdap.py +++ b/ipwhois/rdap.py @@ -28,6 +28,7 @@ from .utils import ipv4_lstrip_zeros, calculate_cidr, unique_everseen from .net import ip_address import logging import json +from collections import namedtuple log = logging.getLogger(__name__) @@ -688,9 +689,95 @@ class RDAP: raise NetError('The provided net parameter is not an instance of ' 'ipwhois.net.Net') + def _get_entity(self, entity=None, roles=None, inc_raw=False, retry_count=3, + asn_data=None, bootstrap=False, rate_limit_timeout=120): + """ + The function for retrieving and parsing information for an entity via + RDAP (HTTP). + + Args: + entity (:obj:`str`): The entity name to lookup. + roles (:obj:`dict`): The mapping of entity handles to roles. + inc_raw (:obj:`bool`, optional): Whether to include the raw + results in the returned dictionary. Defaults to False. + retry_count (:obj:`int`): The number of times to retry in case + socket errors, timeouts, connection resets, etc. are + encountered. Defaults to 3. + asn_data (:obj:`dict`): Result from + :obj:`ipwhois.asn.IPASN.lookup`. Optional if the bootstrap + parameter is True. + bootstrap (:obj:`bool`): If True, performs lookups via ARIN + bootstrap rather than lookups based on ASN data. Defaults to + False. + rate_limit_timeout (:obj:`int`): The number of seconds to wait + before retrying when a rate limit notice is returned via + rdap+json. Defaults to 120. + + Returns: + namedtuple: + + :result (dict): Consists of the fields listed in the + ipwhois.rdap._RDAPEntity dict. The raw result is included for + each object if the inc_raw parameter is True. + :roles (dict): The mapping of entity handles to roles. + """ + + result = {} + + if bootstrap: + entity_url = '{0}/entity/{1}'.format( + BOOTSTRAP_URL, entity) + else: + tmp_reg = asn_data['asn_registry'] + entity_url = RIR_RDAP[tmp_reg]['entity_url'] + entity_url = str(entity_url).format(entity) + + try: + + # RDAP entity query + response = self._net.get_http_json( + url=entity_url, retry_count=retry_count, + rate_limit_timeout=rate_limit_timeout + ) + + # Parse the entity + result_ent = _RDAPEntity(response) + result_ent.parse() + result = result_ent.vars + + result['roles'] = None + try: + + result['roles'] = roles[entity] + + except KeyError: # pragma: no cover + + pass + + try: + + for tmp in response['entities']: + + if tmp['handle'] not in roles: + roles[tmp['handle']] = tmp['roles'] + + except (IndexError, KeyError): + + pass + + if inc_raw: + result['raw'] = response + + except (HTTPLookupError, InvalidEntityObject): + + pass + + return_tuple = namedtuple('return_tuple', ['result', 'roles']) + return return_tuple(result, roles) + def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, excluded_entities=None, response=None, bootstrap=False, - rate_limit_timeout=120): + rate_limit_timeout=120, root_ent_check=True): """ The function for retrieving and parsing information for an IP address via RDAP (HTTP). @@ -716,6 +803,9 @@ class RDAP: rate_limit_timeout (:obj:`int`): The number of seconds to wait before retrying when a rate limit notice is returned via rdap+json. Defaults to 120. + root_ent_check (:obj:`bool`): If True, will perform + additional RDAP HTTP queries for missing entity data at the + root level. Defaults to True. Returns: dict: The IP RDAP lookup results @@ -792,10 +882,23 @@ class RDAP: if ent['handle'] not in [results['entities'], excluded_entities]: - result_ent = _RDAPEntity(ent) - result_ent.parse() + if 'vcardArray' not in ent and root_ent_check: + entity_object, roles = self._get_entity( + entity=ent['handle'], + roles=roles, + inc_raw=inc_raw, + retry_count=retry_count, + asn_data=asn_data, + bootstrap=bootstrap, + rate_limit_timeout=rate_limit_timeout + ) + results['objects'][ent['handle']] = entity_object - results['objects'][ent['handle']] = result_ent.vars + else: + result_ent = _RDAPEntity(ent) + result_ent.parse() + + results['objects'][ent['handle']] = result_ent.vars results['entities'].append(ent['handle']) @@ -835,57 +938,18 @@ class RDAP: list(new_objects.keys()) + excluded_entities): - if bootstrap: - entity_url = '{0}/entity/{1}'.format( - BOOTSTRAP_URL, ent) - else: - tmp_reg = asn_data['asn_registry'] - entity_url = RIR_RDAP[tmp_reg]['entity_url'] - entity_url = str(entity_url).format(ent) + entity_object, roles = self._get_entity( + entity=ent, + roles=roles, + inc_raw=inc_raw, + retry_count=retry_count, + asn_data=asn_data, + bootstrap=bootstrap, + rate_limit_timeout=rate_limit_timeout + ) + new_objects[ent] = entity_object - try: - - # RDAP entity query - response = self._net.get_http_json( - url=entity_url, retry_count=retry_count, - rate_limit_timeout=rate_limit_timeout - ) - - # Parse the entity - result_ent = _RDAPEntity(response) - result_ent.parse() - new_objects[ent] = result_ent.vars - - new_objects[ent]['roles'] = None - try: - - new_objects[ent]['roles'] = roles[ent] - - except KeyError: # pragma: no cover - - pass - - try: - - for tmp in response['entities']: - - if tmp['handle'] not in roles: - - roles[tmp['handle']] = tmp['roles'] - - except (IndexError, KeyError): - - pass - - if inc_raw: - - new_objects[ent]['raw'] = response - - except (HTTPLookupError, InvalidEntityObject): - - pass - - except TypeError: + except (KeyError, TypeError): pass diff --git a/ipwhois/tests/test_rdap.py b/ipwhois/tests/test_rdap.py index 2ccdb45..6d4277d 100644 --- a/ipwhois/tests/test_rdap.py +++ b/ipwhois/tests/test_rdap.py @@ -82,7 +82,8 @@ class TestRDAP(TestCommon): 'endAddress': '74.125.225.229' }, asn_data=val['asn_data'], - depth=0), dict) + depth=0, + root_ent_check=False), dict) log.debug('Testing rdap.lookup entitiy checks') net = Net('74.125.225.229') @@ -99,7 +100,8 @@ class TestRDAP(TestCommon): 'entities': entity }, asn_data=val['asn_data'], - depth=1), dict) + depth=0, + root_ent_check=False), dict) self.assertIsInstance(obj.lookup(response={ 'handle': 'test', @@ -109,9 +111,10 @@ class TestRDAP(TestCommon): 'entities': entity }, asn_data=val['asn_data'], - depth=1, + depth=0, bootstrap=True, - inc_raw=True), dict) + inc_raw=True, + root_ent_check=False), dict) # No sub entities. This is for coverage, but won't error out. entity = [{'handle': 'test', 'roles': [ @@ -125,7 +128,8 @@ class TestRDAP(TestCommon): 'entities': entity }, asn_data=val['asn_data'], - depth=1), dict) + depth=0, + root_ent_check=False), dict) class TestRDAPContact(TestCommon): From a54f2ba15f90645f052f36d604392b0bca35aa83 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 15:18:25 -0500 Subject: [PATCH 30/36] Removed old permutations check --- ipwhois/asn.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 60625e2..e2cda71 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -447,11 +447,6 @@ class IPASN: dns_success = False for index, lookup_method in enumerate(lookups): - if index > 0 and not asn_methods: - - raise ASNRegistryError('ASN registry lookup failed. ' - 'Permutations not allowed.') - if lookup_method == 'dns': try: From a0428701e7b04b6d7f82ce7d95e4363e3cfd21df Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 18:01:21 -0500 Subject: [PATCH 31/36] Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236) --- CHANGES.rst | 1 + ipwhois/scripts/ipwhois_utils_cli.py | 49 ++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2b25efe..d9df2fa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -29,6 +29,7 @@ Changelog - Pinned requirements (#274) - Added ip_failed_total key to stats dictionary in experimental.bulk_lookup_rdap (#235) +- Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236) 1.1.0 (2019-02-01) ------------------ diff --git a/ipwhois/scripts/ipwhois_utils_cli.py b/ipwhois/scripts/ipwhois_utils_cli.py index 7f86b95..769823b 100644 --- a/ipwhois/scripts/ipwhois_utils_cli.py +++ b/ipwhois/scripts/ipwhois_utils_cli.py @@ -28,8 +28,9 @@ import argparse from collections import OrderedDict import json from ipwhois.utils import (ipv4_lstrip_zeros, calculate_cidr, get_countries, - ipv4_is_defined, ipv6_is_defined, unique_everseen, - unique_addresses) + ipv4_is_defined, ipv6_is_defined, + ipv4_generate_random, ipv6_generate_random, + unique_everseen, unique_addresses) # CLI ANSI rendering ANSI = { @@ -86,6 +87,22 @@ parser.add_argument( metavar='"IP ADDRESS"', help='Check if an IPv6 address is defined (in a reserved address range).' ) +parser.add_argument( + '--ipv4_generate_random', + type=int, + nargs=1, + metavar='"TOTAL"', + help='Generate random, unique IPv4 addresses that are not defined (can be ' + 'looked up using ipwhois).' +) +parser.add_argument( + '--ipv6_generate_random', + type=int, + nargs=1, + metavar='"TOTAL"', + help='Generate random, unique IPv6 addresses that are not defined (can be ' + 'looked up using ipwhois).' +) parser.add_argument( '--unique_everseen', type=json.loads, @@ -224,6 +241,34 @@ elif script_args.ipv6_is_defined: print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) +elif script_args.ipv4_generate_random: + + try: + + result = ipv4_generate_random(total=script_args.ipv4_generate_random[0]) + + for random_ip in result: + + print(random_ip) + + except Exception as e: + + print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) + +elif script_args.ipv6_generate_random: + + try: + + result = ipv6_generate_random(total=script_args.ipv6_generate_random[0]) + + for random_ip in result: + + print(random_ip) + + except Exception as e: + + print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) + elif script_args.unique_everseen: try: From e7b2013c22ecb07f89d20ea22810a02d897dc98f Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 18:02:31 -0500 Subject: [PATCH 32/36] Fix metavars (#236) --- ipwhois/scripts/ipwhois_utils_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/scripts/ipwhois_utils_cli.py b/ipwhois/scripts/ipwhois_utils_cli.py index 769823b..166ea5d 100644 --- a/ipwhois/scripts/ipwhois_utils_cli.py +++ b/ipwhois/scripts/ipwhois_utils_cli.py @@ -91,7 +91,7 @@ parser.add_argument( '--ipv4_generate_random', type=int, nargs=1, - metavar='"TOTAL"', + metavar='TOTAL', help='Generate random, unique IPv4 addresses that are not defined (can be ' 'looked up using ipwhois).' ) @@ -99,7 +99,7 @@ parser.add_argument( '--ipv6_generate_random', type=int, nargs=1, - metavar='"TOTAL"', + metavar='TOTAL', help='Generate random, unique IPv6 addresses that are not defined (can be ' 'looked up using ipwhois).' ) From ac71f351fff8aa1dfb92882a813abb4eac3ea9f5 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 18:59:00 -0500 Subject: [PATCH 33/36] ipv4_generate_random and ipv6_generate_random docs (#236) --- CLI.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/CLI.rst b/CLI.rst index 03902a2..9c6934f 100644 --- a/CLI.rst +++ b/CLI.rst @@ -168,6 +168,12 @@ optional arguments: --ipv6_is_defined IPADDRESS Check if an IPv6 address is defined (in a reserved address range). + --ipv4_generate_random TOTAL + Generate random, unique IPv4 addresses that are not + defined (can be looked up using ipwhois). + --ipv6_generate_random TOTAL + Generate random, unique IPv6 addresses that are not + defined (can be looked up using ipwhois). --unique_everseen ITERABLE List unique elements from input iterable, preserving the order. @@ -261,6 +267,32 @@ ipv6_is_defined Name: Unique Local Unicast RFC: RFC 4193 +ipv4_generate_random +^^^^^^^^^^^^^^^^^^^^ + +:: + + >>>> ipwhois_utils_cli.py --ipv4_generate_random 5 + + 119.224.47.74 + 128.106.183.195 + 54.97.0.158 + 52.206.105.37 + 126.180.201.81 + +ipv6_generate_random +^^^^^^^^^^^^^^^^^^^^ + +:: + + >>>> ipwhois_utils_cli.py --ipv6_generate_random 5 + + 3e8c:dc93:49c8:57fd:31dd:2963:6332:426e + 2e3d:fd84:b57b:9282:91e6:5d4d:18d5:34f1 + 21d4:9d25:7dd6:e28b:77d7:7ce9:f85f:b34f + 3659:2b9:12ed:1eac:fd40:5756:3753:6d2d + 2e05:6d47:83fd:5de8:c6cb:85cb:912:fdb1 + unique_everseen ^^^^^^^^^^^^^^^ From f499cb1a135985d664ec0b51b1d9a3231d7c7b12 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 19:35:36 -0500 Subject: [PATCH 34/36] Added GitHub issues badge --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 712c939..524bcd7 100644 --- a/README.rst +++ b/README.rst @@ -7,8 +7,10 @@ ipwhois .. image:: https://coveralls.io/repos/github/secynic/ipwhois/badge.svg?branch= master :target: https://coveralls.io/github/secynic/ipwhois?branch=master +.. image:: https://img.shields.io/github/issues-raw/secynic/ipwhois + :target: https://github.com/secynic/ipwhois/issues .. image:: https://codeclimate.com/github/secynic/ipwhois/badges/issue_count.svg - :target: https://codeclimate.com/github/secynic/ipwhois + :target: https://codeclimate.com/github/secynic/ipwhois .. image:: https://img.shields.io/badge/license-BSD%202--Clause-blue.svg :target: https://github.com/secynic/ipwhois/tree/master/LICENSE.txt .. image:: https://img.shields.io/badge/python-2.7%2C%203.4+-blue.svg From 429fb0cee6fd23f3f24de0300da815f527f23640 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 19:50:21 -0500 Subject: [PATCH 35/36] Added documentation note for ASN data (#278) --- ASN.rst | 19 +++++++++++++++++++ CHANGES.rst | 1 + 2 files changed, 20 insertions(+) diff --git a/ASN.rst b/ASN.rst index bcfed90..23691ed 100644 --- a/ASN.rst +++ b/ASN.rst @@ -5,6 +5,25 @@ IP ASN Lookups This is new functionality as of v0.15.0. This functionality was migrated from net.Net and is still used by IPWhois.lookup*(). +.. note:: + + Cymru ASN data should not be considered a primary source for data points + like country code. + + Message from the Cymru site:: + + The country code, registry, and allocation date are all based on data + obtained directly from the regional registries including: ARIN, RIPE, + AFRINIC, APNIC, LACNIC. The information returned relating to these + categories will only be as accurate as the data present in the RIR + databases. + + IMPORTANT NOTE: Country codes are likely to vary significantly from + actual IP locations, and we must strongly advise that the IP to ASN + mapping tool not be used as an IP geolocation (GeoIP) service. + + https://team-cymru.com/community-services/ip-asn-mapping/ + .. _ip-asn-input: IP ASN Input diff --git a/CHANGES.rst b/CHANGES.rst index d9df2fa..c70c030 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -30,6 +30,7 @@ Changelog - Added ip_failed_total key to stats dictionary in experimental.bulk_lookup_rdap (#235) - Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236) +- Added documentation note for ASN data (#278) 1.1.0 (2019-02-01) ------------------ From 50df6efb9631cea7e9db38c5e31e0a6d0dc9ba33 Mon Sep 17 00:00:00 2001 From: secynic Date: Thu, 17 Sep 2020 09:45:15 -0500 Subject: [PATCH 36/36] Fixed NIR datetime parsing issue if only date is returned (#284) --- CHANGES.rst | 1 + ipwhois/nir.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c70c030..8bcb171 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,6 +21,7 @@ Changelog - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) - Fixed bug in root and sub-entities not getting queried/data (#247) +- Fixed NIR datetime parsing issue if only date is returned (#284) - Added new argument root_ent_check to IPWhois.lookup_rdap and RDAP.lookup. Set this to False to revert to old functionality - missing data, but less queries (#247) diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 3d2f603..231f379 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -265,12 +265,20 @@ class NIRWhois: if field in ['created', 'updated'] and dt_format: - value = ( - datetime.strptime( - values[0], - str(dt_format) - ) - timedelta(hours=hourdelta) - ).isoformat('T') + try: + value = ( + datetime.strptime( + values[0], + str(dt_format) + ) - timedelta(hours=hourdelta) + ).isoformat('T') + except ValueError: + value = ( + datetime.strptime( + values[0], + '%Y/%m/%d' + ) + ).isoformat('T') elif field in ['nameservers']: