1. import re
  2. from urllib.parse import urlparse
  3. import requests
  5. def validate_url(url):
  6. """
  7. Validates a URL and checks for basic safety.
  8. """
  9. try:
  10. result = urlparse(url)
  11. if not all([result.scheme, result.netloc]):
  12. return False, "Invalid URL format"
  14. if result.scheme not in ["http", "https"]:
  15. return False, "Unsupported scheme"
  17. return True, None
  18. except Exception as e:
  19. return False, f"URL parsing error: {str(e)}"
  21. def check_url_safety(url):
  22. """
  23. Checks if a URL is potentially malicious based on heuristics.
  24. """
  25. # Basic checks
  26. if "http://example.com" in url:
  27. return False, "URL points to example.com (safe)"
  28. if "http://localhost" in url:
  29. return False, "URL points to localhost (potentially safe, but check context)"
  30. if "http://127.0.0.1" in url:
  31. return False, "URL points to 127.0.0.1 (potentially safe, but check context)"
  33. # Check for suspicious characters (e.g., excessive dots, unusual characters)
  34. if re.search(r"[.]{5,}", url):
  35. return False, "URL contains excessive dots"
  36. if re.search(r"[^a-zA-Z0-9._~:\/-]", url):
  37. return False, "URL contains potentially malicious characters"
  39. return True, None
  42. def flag_anomalous_urls(url_list, threshold=0.8):
  43. """
  44. Flags anomalous URLs in a list based on validation and safety checks.
  46. Args:
  47. url_list: A list of URLs to check.
  48. threshold: A threshold for the overall anomaly score. If the score
  49. exceeds the threshold, the URL is flagged.
  51. Returns:
  52. A list of flagged URLs and their corresponding reasons.
  53. """
  54. flagged_urls = []
  55. total_urls = len(url_list)
  57. for url in url_list:
  58. is_valid, validation_error = validate_url(url)
  59. if not is_valid:
  60. flagged_urls.append({"url": url, "reason": validation_error})
  61. continue # Skip further checks if URL is invalid
  63. is_safe, safety_error = check_url_safety(url)
  64. if not is_safe:
  65. flagged_urls.append({"url": url, "reason": safety_error})
  66. continue
  68. # Add more sophisticated checks here (e.g., checking website content)
  69. # Example: Check if the website exists and returns a valid HTTP status code
  70. try:
  71. response = requests.get(url, timeout=5)
  72. response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
  73. except requests.exceptions.RequestException as e:
  74. flagged_urls.append({"url": url, "reason": f"Website unreachable: {str(e)}"})
  75. continue
  78. # Calculate an overall anomaly score (simple example)
  79. total_anomalies = len(flagged_urls)
  80. if total_urls > 0:
  81. anomaly_score = total_anomalies / total_urls
  82. else:
  83. anomaly_score = 0.0
  85. if anomaly_score > threshold:
  86. print(f"Anomaly score ({anomaly_score:.2f}) exceeds threshold ({threshold}).")
  87. print("Flagged URLs:")
  88. for item in flagged_urls:
  89. print(f" - {item['url']}: {item['reason']}")
  93. return flagged_urls
  96. if __name__ == '__main__':
  97. # Example usage:
  98. url_list = [
  99. "https://www.google.com",
  100. "http://example.com",
  101. "http://malicious.example.com/evil.php",
  102. "invalid-url",
  103. "https://www.example.com/path with spaces",
  104. "http://127.0.0.1",
  105. "http://localhost:8000",
  106. "http://example.com/a..b",
  107. "https://verylongdomainnamewithmanydots.com",
  108. ]
  110. flagged = flag_an

Add your comment