From e6f9013e9913629eecf2103718f450c040d46ed0 Mon Sep 17 00:00:00 2001 From: Slava Olesik Date: Sat, 24 Oct 2026 12:13:58 +0300 Subject: [PATCH 1/3] phishing_detection: web_traffic feature from the NetAPI Top 1M instead of the dead Alexa endpoint --- .../phishing_detection/features_extraction.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Cybersecurity_Tools/phishing_detection/features_extraction.py b/Cybersecurity_Tools/phishing_detection/features_extraction.py index 5e432064b0..d4ee344250 100644 --- a/Cybersecurity_Tools/phishing_detection/features_extraction.py +++ b/Cybersecurity_Tools/phishing_detection/features_extraction.py @@ -257,10 +257,23 @@ def check_dns_record(self): self.features['DNSRecord'] = -1 def check_web_traffic(self): - """Check web traffic (simplified)""" + """Check web traffic: position of the domain in the NetAPI Top 1M (the Alexa ranking was retired in 2022). + Same scale as the training data: 1 = rank up to 100,000, 0 = ranked lower in the Top 1M, -1 = not ranked.""" try: - response = requests.get(f"https://data.alexa.com/data?cli=10&url={self.domain}") - self.features['web_traffic'] = 1 if response.status_code == 200 else -1 + ext = tldextract.extract(self.url) + registered_domain = f"{ext.domain}.{ext.suffix}" if ext.suffix else self.domain + response = requests.get( + "https://netapi.com/api-json/", + params={"method": "domain-rank", "domain": registered_domain}, + timeout=10, + ) + rank = response.json().get("rank") if response.status_code == 200 else None + if rank is None: + self.features['web_traffic'] = -1 + elif rank <= 100000: + self.features['web_traffic'] = 1 + else: + self.features['web_traffic'] = 0 except: self.features['web_traffic'] = -1 From e8302ddd355ef676285ab5a8bcdb6e2f303c9074 Mon Sep 17 00:00:00 2001 From: Slava Olesik Date: Sun, 25 Oct 2026 11:05:57 +0200 Subject: [PATCH 2/3] web_traffic: keep tenant hostnames of private suffixes, cache ranks, wait out the per-minute limit --- .../phishing_detection/features_extraction.py | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/Cybersecurity_Tools/phishing_detection/features_extraction.py b/Cybersecurity_Tools/phishing_detection/features_extraction.py index d4ee344250..269206edec 100644 --- a/Cybersecurity_Tools/phishing_detection/features_extraction.py +++ b/Cybersecurity_Tools/phishing_detection/features_extraction.py @@ -1,5 +1,6 @@ import re import socket +import time import ssl import whois import requests @@ -9,6 +10,35 @@ import dns.resolver import tldextract +# NetAPI domain-rank answers shared by all extractor instances: one request per registered domain +_RANK_CACHE = {} + + +def netapi_domain_rank(registered_domain): + """Rank of a registered domain in the NetAPI Top 1M, None when unranked or when the lookup failed. + Free calls without a key are limited to 30 a minute and 1,000 a day per IP: a per-minute 429 is waited + out once (Retry-After), the daily limit is not, and failed lookups are not cached.""" + if registered_domain in _RANK_CACHE: + return _RANK_CACHE[registered_domain] + for attempt in range(2): + response = requests.get( + "https://netapi.com/api-json/", + params={"method": "domain-rank", "domain": registered_domain}, + timeout=10, + ) + if response.status_code == 429 and attempt == 0: + retry_after = int(response.headers.get("Retry-After", "60")) + if retry_after <= 60: + time.sleep(retry_after) + continue + break + if response.status_code != 200: + return None + rank = response.json().get("rank") + _RANK_CACHE[registered_domain] = rank + return rank + + class URLFeatureExtractor: def __init__(self, url): self.url = url @@ -260,14 +290,11 @@ def check_web_traffic(self): """Check web traffic: position of the domain in the NetAPI Top 1M (the Alexa ranking was retired in 2022). Same scale as the training data: 1 = rank up to 100,000, 0 = ranked lower in the Top 1M, -1 = not ranked.""" try: - ext = tldextract.extract(self.url) + # private suffixes (github.io, blogspot.com ...) keep the tenant name, so phish.github.io + # is looked up as itself and does not inherit the rank of the platform + ext = tldextract.extract(self.url, include_psl_private_domains=True) registered_domain = f"{ext.domain}.{ext.suffix}" if ext.suffix else self.domain - response = requests.get( - "https://netapi.com/api-json/", - params={"method": "domain-rank", "domain": registered_domain}, - timeout=10, - ) - rank = response.json().get("rank") if response.status_code == 200 else None + rank = netapi_domain_rank(registered_domain) if rank is None: self.features['web_traffic'] = -1 elif rank <= 100000: From d7a805ee466fefdc1f3a117c67a7359fce4957f7 Mon Sep 17 00:00:00 2001 From: Slava Olesik Date: Mon, 26 Oct 2026 10:13:46 +0200 Subject: [PATCH 3/3] web_traffic: expire cached ranks after six hours (the Top 1M is rebuilt daily) --- .../phishing_detection/features_extraction.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Cybersecurity_Tools/phishing_detection/features_extraction.py b/Cybersecurity_Tools/phishing_detection/features_extraction.py index 269206edec..2fc47dc90f 100644 --- a/Cybersecurity_Tools/phishing_detection/features_extraction.py +++ b/Cybersecurity_Tools/phishing_detection/features_extraction.py @@ -10,16 +10,19 @@ import dns.resolver import tldextract -# NetAPI domain-rank answers shared by all extractor instances: one request per registered domain +# NetAPI domain-rank answers shared by all extractor instances: one request per registered domain. +# The Top 1M is rebuilt daily, so an entry expires after six hours and a long-running process picks up new ranks. _RANK_CACHE = {} +_RANK_CACHE_TTL = 6 * 3600 def netapi_domain_rank(registered_domain): """Rank of a registered domain in the NetAPI Top 1M, None when unranked or when the lookup failed. Free calls without a key are limited to 30 a minute and 1,000 a day per IP: a per-minute 429 is waited out once (Retry-After), the daily limit is not, and failed lookups are not cached.""" - if registered_domain in _RANK_CACHE: - return _RANK_CACHE[registered_domain] + cached = _RANK_CACHE.get(registered_domain) + if cached is not None and time.monotonic() - cached[1] < _RANK_CACHE_TTL: + return cached[0] for attempt in range(2): response = requests.get( "https://netapi.com/api-json/", @@ -35,7 +38,7 @@ def netapi_domain_rank(registered_domain): if response.status_code != 200: return None rank = response.json().get("rank") - _RANK_CACHE[registered_domain] = rank + _RANK_CACHE[registered_domain] = (rank, time.monotonic()) return rank