1. import time
  2. import random
  3. import json
  5. def generate_metadata(num_records=1000):
  6. """Generates sample metadata records."""
  7. metadata = []
  8. for i in range(num_records):
  9. metadata.append({
  10. "id": i,
  11. "name": f"Item {i}",
  12. "description": "A sample item",
  13. "category": random.choice(["Electronics", "Clothing", "Books"]),
  14. "price": round(random.uniform(10, 1000), 2),
  15. "valid": True # Initially assume valid
  16. })
  17. return metadata
  19. def validate_metadata(metadata, validation_rules):
  20. """Validates metadata against defined rules."""
  21. invalid_records = []
  22. for record in metadata:
  23. is_valid = True
  24. for rule_name, rule_value in validation_rules.items():
  25. if rule_name == "price_min":
  26. if record["price"] < rule_value:
  27. is_valid = False
  28. break
  29. elif rule_name == "category_allowed":
  30. if record["category"] not in rule_value:
  31. is_valid = False
  32. break
  33. # Add more validation rules here
  34. if not is_valid:
  35. invalid_records.append(record)
  36. return invalid_records
  38. def apply_manual_overrides(metadata, overrides):
  39. """Applies manual overrides to metadata."""
  40. for record_id, override_data in overrides.items():
  41. for key, value in override_data.items():
  42. if key in metadata[record_id]:
  43. metadata[record_id][key] = value
  44. return metadata
  46. def measure_performance(metadata, validation_rules, overrides):
  47. """Measures the performance of metadata validation with overrides."""
  48. start_time = time.time()
  50. # Validation
  51. invalid_records = validate_metadata(metadata, validation_rules)
  52. validation_time = time.time() - start_time
  54. # Apply overrides
  55. updated_metadata = apply_manual_overrides(metadata, overrides)
  56. override_time = time.time() - start_time
  58. return validation_time, override_time, invalid_records, updated_metadata
  61. if __name__ == "__main__":
  62. # Define validation rules
  63. validation_rules = {
  64. "price_min": 50,
  65. "category_allowed": ["Electronics", "Clothing"]
  66. }
  68. # Define manual overrides
  69. overrides = {
  70. 0: {"price": 150, "description": "Updated description"}
  71. }
  73. # Generate sample metadata
  74. metadata = generate_metadata(1000)
  76. # Measure performance
  77. validation_time, override_time, invalid_records, updated_metadata = measure_performance(
  78. metadata, validation_rules, overrides
  79. )
  81. print(f"Validation Time: {validation_time:.4f} seconds")
  82. print(f"Override Time: {override_time:.4f} seconds")
  83. print(f"Number of Invalid Records: {len(invalid_records)}")
  85. # Optionally, save the updated metadata to a JSON file
  86. with open("updated_metadata.json", "w") as f:
  87. json.dump(updated_metadata, f, indent=4)

Add your comment