1. from bs4 import BeautifulSoup
  2. import requests
  3. import re
  5. def clean_html(html_content):
  6. """
  7. Cleans HTML content for monitoring purposes.
  8. Removes script, style, and potentially irrelevant tags/attributes.
  9. """
  10. soup = BeautifulSoup(html_content, 'html.parser')
  12. # Remove script and style tags
  13. for script in soup(["script", "style"]):
  14. script.extract()
  16. # Remove comments
  17. for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
  18. comment.extract()
  20. # Remove potentially irrelevant tags - adjust the list as needed
  21. tags_to_remove = ['nav', 'footer', 'header', 'aside']
  22. for tag in tags_to_remove:
  23. for element in soup.find_all(tag):
  24. element.extract()
  26. # Remove extra whitespace and newlines
  27. text = soup.get_text()
  28. text = re.sub(r'\s+', ' ', text).strip() #replace multiple spaces with single space
  30. return text
  34. def fetch_and_clean_html(url):
  35. """
  36. Fetches HTML from a URL and cleans it.
  37. """
  38. try:
  39. response = requests.get(url)
  40. response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
  41. html_content = response.text
  42. cleaned_text = clean_html(html_content)
  43. return cleaned_text
  44. except requests.exceptions.RequestException as e:
  45. print(f"Error fetching URL {url}: {e}")
  46. return None
  50. if __name__ == '__main__':
  51. # Example Usage (replace with your monitoring URLs)
  52. url_list = [
  53. "https://www.example.com",
  54. "https://www.python.org"
  55. ]
  57. for url in url_list:
  58. cleaned_data = fetch_and_clean_html(url)
  59. if cleaned_data:
  60. print(f"Cleaned data from {url}:\n{cleaned_data[:500]}...\n") #print first 500 chars

Add your comment