1. import re
  2. import logging
  4. # Configure logging
  5. logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
  7. def normalize_text(input_file, output_file):
  8. """
  9. Normalizes text data from an input file, addressing common issues like:
  10. - Lowercasing
  11. - Removing punctuation
  12. - Removing extra whitespace
  13. - Replacing special characters
  14. - Error logging for problematic lines.
  15. """
  16. try:
  17. with open(input_file, 'r', encoding='utf-8') as infile, open(output_file, 'w', encoding='utf-8') as outfile:
  18. for line_num, line in enumerate(infile, 1): # Enumerate for line numbers
  19. try:
  20. # Lowercase the line
  21. line = line.lower()
  23. # Remove punctuation
  24. line = re.sub(r'[^\w\s]', '', line)
  26. # Remove extra whitespace
  27. line = re.sub(r'\s+', ' ', line).strip()
  29. # Replace special characters (example: keep only alphanumeric and spaces)
  30. line = re.sub(r'[^a-z0-9\s]', '', line)
  32. # Write the normalized line to the output file
  33. outfile.write(line)
  35. except Exception as e:
  36. logging.error(f"Error processing line {line_num}: {e}. Line: {line.strip()}")
  38. except FileNotFoundError:
  39. logging.error(f"Input file not found: {input_file}")
  40. except Exception as e:
  41. logging.error(f"An unexpected error occurred: {e}")
  45. if __name__ == '__main__':
  46. # Example usage:
  47. input_file = 'input.txt' # Replace with your input file
  48. output_file = 'output.txt' # Replace with your desired output file
  49. normalize_text(input_file, output_file)
  50. print(f"Text normalization complete. Output written to {output_file}")

Add your comment