1. import urllib.request
  2. import urllib.parse
  3. import ssl
  4. import socket
  5. import threading
  6. import queue
  8. class CompatHTTPClient:
  9. def __init__(self, max_memory=1024*1024): # 1MB default
  10. self.max_memory = max_memory
  11. self.request_queue = queue.Queue()
  12. self.response_queue = queue.Queue()
  13. self.running = True
  14. self.request_threads = []
  15. self.response_threads = []
  17. def request(self, url, method='GET', headers=None, data=None, timeout=None):
  18. """
  19. Sends an HTTP request and returns the response.
  20. Handles memory constraints by streaming the response.
  21. """
  22. req = urllib.request.Request(url, method=method, headers=headers)
  23. try:
  24. with urllib.request.urlopen(req, timeout=timeout) as response:
  25. # Stream the response
  26. for chunk in iter(lambda: response.read(4096), b''):
  27. yield chunk
  28. except Exception as e:
  29. yield f"Error: {e}"
  32. def submit_request(self, url, method='GET', headers=None, data=None, timeout=None):
  33. """
  34. Submits a request to the request queue.
  35. """
  36. self.request_queue.put((url, method, headers, data, timeout))
  38. def _worker_request(self):
  39. """
  40. Worker thread to handle requests from the queue.
  41. """
  42. while self.running:
  43. try:
  44. url, method, headers, data, timeout = self.request_queue.get(timeout=0.1)
  45. try:
  46. response = self.request(url, method, headers, data, timeout)
  47. self.response_queue.put((url, response))
  48. except Exception as e:
  49. self.response_queue.put((url, f"Error: {e}"))
  50. finally:
  51. self.request_queue.task_done()
  52. except queue.Empty:
  53. pass # Continue if queue is empty
  55. def _worker_response(self):
  56. """
  57. Worker thread to handle responses from the queue.
  58. """
  59. while self.running:
  60. try:
  61. url, response = self.response_queue.get(timeout=0.1)
  62. yield url, response
  63. except queue.Empty:
  64. pass # Continue if queue is empty
  66. def start(self, num_request_threads=4, num_response_threads=4):
  67. """
  68. Starts the worker threads.
  69. """
  70. for _ in range(num_request_threads):
  71. thread = threading.Thread(target=self._worker_request)
  72. thread.daemon = True
  73. self.request_threads.append(thread)
  74. thread.start()
  76. for _ in range(num_response_threads):
  77. thread = threading.Thread(target=self._worker_response)
  78. thread.daemon = True
  79. self.response_threads.append(thread)
  80. thread.start()
  82. def stop(self):
  83. """
  84. Stops the worker threads.
  85. """
  86. self.running = False
  87. for thread in self.request_threads:
  88. thread.join(timeout=0.1)
  89. for thread in self.response_threads:
  90. thread.join(timeout=0.1)

Add your comment