1. import requests
  2. from urllib.parse import urlparse
  3. from collections import defaultdict
  5. def remove_duplicate_requests(request_list):
  6. """
  7. Removes duplicate HTTP requests from a list, considering URL and method.
  9. Args:
  10. request_list: A list of dictionaries, where each dictionary represents an HTTP request
  11. and contains at least 'url' and 'method' keys.
  13. Returns:
  14. A list of unique HTTP requests.
  15. """
  16. seen_requests = set() # Use a set for efficient duplicate checking
  17. unique_requests = []
  19. for request in request_list:
  20. url = request['url']
  21. method = request['method']
  23. # Create a tuple of (url, method) for hashing
  24. request_tuple = (url, method)
  26. if request_tuple not in seen_requests:
  27. unique_requests.append(request)
  28. seen_requests.add(request_tuple)
  30. return unique_requests
  32. if __name__ == '__main__':
  33. # Example usage:
  34. sample_requests = [
  35. {'url': 'https://www.example.com', 'method': 'GET'},
  36. {'url': 'https://www.example.com', 'method': 'GET'}, # Duplicate
  37. {'url': 'https://www.google.com', 'method': 'GET'},
  38. {'url': 'https://www.example.com/path', 'method': 'POST'},
  39. {'url': 'https://www.google.com', 'method': 'GET'}, # Duplicate
  40. {'url': 'https://www.example.com', 'method': 'POST'}
  41. ]
  43. unique_requests = remove_duplicate_requests(sample_requests)
  45. for request in unique_requests:
  46. print(request)

Add your comment