1. import java.util.HashSet;
  2. import java.util.List;
  3. import java.util.Set;
  4. import java.util.ArrayList;
  6. public class RecordDeduplicator {
  8. /**
  9. * Deduplicates a list of records, considering a set of fields for uniqueness.
  10. * This is designed for dry-run scenarios with rate limiting.
  11. *
  12. * @param records The list of records to deduplicate. Each record is assumed to be an object with fields.
  13. * @param fieldExtractor A function that extracts the unique identifier fields from each record.
  14. * @return A new list containing only the unique records.
  15. */
  16. public static List<Object> deduplicate(List<?> records, FieldExtractor fieldExtractor) {
  17. Set<String> seenKeys = new HashSet<>(); // Use a Set to efficiently track seen keys.
  18. List<Object> uniqueRecords = new ArrayList<>();
  20. for (Object record : records) {
  21. String key = fieldExtractor.extractKey(record); // Extract the unique key from the record.
  23. if (key == null) {
  24. // Handle cases where key extraction fails. Could log an error.
  25. continue; // Skip this record.
  26. }
  28. if (!seenKeys.contains(key)) {
  29. seenKeys.add(key); // Mark the key as seen.
  30. uniqueRecords.add(record); // Add the record to the unique list.
  31. }
  32. }
  34. return uniqueRecords;
  35. }
  37. /**
  38. * Functional interface to extract the unique key from a record.
  39. */
  40. public interface FieldExtractor {
  41. String extractKey(Object record);
  42. }
  45. public static void main(String[] args) {
  46. // Example Usage (Dry-Run)
  47. List<Object> data = new ArrayList<>();
  48. data.add(new Record("1", "Alice", "123"));
  49. data.add(new Record("2", "Bob", "456"));
  50. data.add(new Record("1", "Alice", "123")); // Duplicate
  51. data.add(new Record("3", "Charlie", "789"));
  52. data.add(new Record("2", "Bob", "456")); // Duplicate
  54. FieldExtractor extractor = record -> {
  55. //Extract unique key based on ID field.
  56. return ((Record)record).id;
  57. };
  59. List<Object> deduplicatedData = deduplicate(data, extractor);
  61. System.out.println("Original Data: " + data);
  62. System.out.println("Deduplicated Data: " + deduplicatedData);
  63. }
  65. //Simple record class for testing.
  66. static class Record {
  67. String id;
  68. String name;
  69. String value;
  71. public Record(String id, String name, String value) {
  72. this.id = id;
  73. this.name = name;
  74. this.value = value;
  75. }
  76. }
  77. }

Add your comment