1. import requests
  2. import time
  3. import logging
  5. # Configure logging
  6. logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s')
  8. def batch_http_requests(url_list):
  9. """
  10. Processes a list of URLs by making HTTP requests, handling potential failures.
  12. Args:
  13. url_list: A list of URLs to process.
  15. Returns:
  16. A list of results, where each result is a tuple containing the URL and its response.
  17. If a request fails, the result will contain the URL and an error message.
  18. """
  19. results = []
  20. for url in url_list:
  21. try:
  22. response = requests.get(url, timeout=10) # Add a timeout
  23. response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
  24. results.append((url, response.text))
  25. print(f"Successfully processed {url}") # Indicate success
  26. except requests.exceptions.RequestException as e:
  27. logging.error(f"Error processing {url}: {e}") # Log the error
  28. results.append((url, str(e)))
  29. print(f"Failed to process {url}: {e}") # Indicate failure
  30. time.sleep(0.5) # Add a delay between requests
  31. return results
  33. if __name__ == '__main__':
  34. # Example usage
  35. urls = [
  36. "https://www.example.com",
  37. "https://httpstat.us/200",
  38. "https://httpstat.us/404",
  39. "https://httpstat.us/500",
  40. "https://invalid-url.example" # Simulate a connection error
  41. ]
  43. results = batch_http_requests(urls)
  45. for url, result in results:
  46. print(f"URL: {url}, Result: {result[:50]}...") # Print the result

Add your comment