1. import requests
  2. from urllib.parse import urlparse
  4. def fetch_data_with_cookie_fallback(url, cookie_manager=None, max_retries=3):
  5. """
  6. Fetches data from a URL, attempting to use cookies if available.
  7. Falls back to a plain request if cookies are not present or if the request fails.
  9. Args:
  10. url (str): The URL to fetch data from.
  11. cookie_manager (object, optional): An object that manages cookies. Defaults to None.
  12. max_retries (int, optional): Maximum number of retry attempts. Defaults to 3.
  14. Returns:
  15. requests.Response or None: The response object if successful, None otherwise.
  16. """
  17. for attempt in range(max_retries):
  18. try:
  19. # Try to use cookies
  20. if cookie_manager:
  21. response = requests.get(url, cookies=cookie_manager.get_cookies())
  22. else:
  23. response = requests.get(url) # No cookies provided
  24. response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
  25. return response
  27. except requests.exceptions.RequestException as e:
  28. print(f"Attempt {attempt + 1} failed: {e}")
  29. if attempt < max_retries - 1:
  30. print("Retrying...")
  31. continue
  32. else:
  33. print("Failed after multiple retries.")
  34. return None
  36. except Exception as e:
  37. print(f"An unexpected error occurred: {e}")
  38. return None
  40. if __name__ == '__main__':
  41. #Example usage
  42. url = "https://httpbin.org/cookies/set?session_id=12345" #URL with cookies
  43. #url = "https://httpbin.org/get" #URL without cookies
  45. # Simulate a cookie manager
  46. class SimpleCookieManager:
  47. def __init__(self):
  48. self.cookies = {}
  50. def get_cookies(self):
  51. return self.cookies
  53. def set_cookie(self, name, value):
  54. self.cookies[name] = value
  56. cookie_manager = SimpleCookieManager()
  57. cookie_manager.set_cookie("session_id", "12345")
  59. response = fetch_data_with_cookie_fallback(url, cookie_manager)
  61. if response:
  62. print("Successfully fetched data:")
  63. print(response.text)
  64. else:
  65. print("Failed to fetch data.")

Add your comment