1. import pandas as pd
  3. def validate_data(df, validation_rules, overrides=None):
  4. """
  5. Validates data in a Pandas DataFrame against predefined rules, allowing manual overrides.
  7. Args:
  8. df (pd.DataFrame): The DataFrame to validate.
  9. validation_rules (dict): A dictionary defining validation rules for each column.
  10. Example: {'column_name': {'type': 'int', 'min': 0, 'max': 100}}
  11. overrides (dict, optional): A dictionary of specific overrides for individual rows.
  12. Keys are row indices, values are dictionaries of overrides.
  13. Defaults to None.
  15. Returns:
  16. pd.DataFrame: The validated DataFrame. Rows that fail validation are flagged with a 'validation_error' column.
  17. """
  19. df['validation_error'] = False # Initialize validation error column
  21. for col, rules in validation_rules.items():
  22. col_errors = [] # Store errors for each column
  24. for index, row in df.iterrows():
  25. value = row[col]
  27. if value is None:
  28. if rules.get('required', False):
  29. col_errors.append(f"Column '{col}' is required")
  30. df.loc[index, 'validation_error'] = True
  31. continue # Skip to next row if value is None and not required
  33. if rules['type'] == 'int':
  34. try:
  35. value = int(value)
  36. except ValueError:
  37. col_errors.append(f"Column '{col}' must be an integer")
  38. df.loc[index, 'validation_error'] = True
  39. continue # Skip if not an integer
  40. if 'min' in rules and value < rules['min']:
  41. col_errors.append(f"Column '{col}' must be >= {rules['min']}")
  42. df.loc[index, 'validation_error'] = True
  43. if 'max' in rules and value > rules['max']:
  44. col_errors.append(f"Column '{col}' must be <= {rules['max']}")
  45. df.loc[index, 'validation_error'] = True
  47. elif rules['type'] == 'float':
  48. try:
  49. value = float(value)
  50. except ValueError:
  51. col_errors.append(f"Column '{col}' must be a float")
  52. df.loc[index, 'validation_error'] = True
  53. continue
  55. if 'min' in rules and value < rules['min']:
  56. col_errors.append(f"Column '{col}' must be >= {rules['min']}")
  57. df.loc[index, 'validation_error'] = True
  58. if 'max' in rules and value > rules['max']:
  59. col_errors.append(f"Column '{col}' must be <= {rules['max']}")
  60. df.loc[index, 'validation_error'] = True
  62. elif rules['type'] == 'string':
  63. value = str(value).strip()
  64. if 'min_length' in rules and len(value) < rules['min_length']:
  65. col_errors.append(f"Column '{col}' must be at least {rules['min_length']} characters")
  66. df.loc[index, 'validation_error'] = True
  67. if 'max_length' in rules and len(value) > rules['max_length']:
  68. col_errors.append(f"Column '{col}' must be at most {rules['max_length']} characters")
  69. df.loc[index, 'validation_error'] = True
  70. if 'pattern' in rules and not re.match(rules['pattern'], value):
  71. col_errors.append(f"Column '{col}' must match pattern: {rules['pattern']}")
  72. df.loc[index, 'validation_error'] = True
  74. if col_errors:
  75. df.loc[index, 'validation_error'] = col_errors # Store all errors in a single cell
  77. if col_errors:
  78. print(f"Validation errors for column '{col}': {col_errors}")
  80. # Apply overrides
  81. if overrides:
  82. for index, override in overrides.items():
  83. row = df.loc[index]
  84. for col, value in override.items():
  85. if col in validation_rules:
  86. df.loc[index, col] = value
  87. return df
  88. import re #Import Regular expressions for string validation
  89. if __

Add your comment