1. import re
  3. def tokenize_message_queue(message):
  4. """
  5. Tokenizes a message queue string.
  7. Args:
  8. message (str): The message queue string.
  10. Returns:
  11. list: A list of tokens.
  12. """
  13. # Define regex patterns for different tokens
  14. patterns = [
  15. r"([a-zA-Z0-9]+)", # Alphanumeric tokens
  16. r"(\d+)", # Numeric tokens
  17. r"([._-]+)", # Dot, underscore or hyphen separated tokens
  18. r"(\s+)", # Whitespace
  19. r"([`~@#$%^&*()_+=\[\]{};':\",./<>?\|])" # Special characters
  20. ]
  22. tokens = []
  23. for pattern in patterns:
  24. tokens.extend(re.findall(pattern, message))
  26. #Remove empty strings
  27. tokens = [token for token in tokens if token]
  29. return tokens
  31. if __name__ == '__main__':
  32. # Example usage
  33. message = "message.queue_name@example.com 123 abc ~def"
  34. tokens = tokenize_message_queue(message)
  35. print(tokens)

Add your comment