1. <?php
  3. /**
  4. * Deduplicates records from a file, performing basic sanity checks.
  5. *
  6. * @param string $inputFile Path to the input file.
  7. * @param string $outputFile Path to the output file.
  8. * @param array $fields Array of fields to use for deduplication. Defaults to all fields.
  9. * @return bool True on success, false on failure.
  10. */
  11. function deduplicateFile(string $inputFile, string $outputFile, array $fields = []): bool
  12. {
  13. if (!file_exists($inputFile)) {
  14. error_log("Input file not found: " . $inputFile);
  15. return false;
  16. }
  18. $lines = file($inputFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
  20. if ($lines === false) {
  21. error_log("Failed to read input file: " . $inputFile);
  22. return false;
  23. }
  25. $seen = [];
  26. $deduplicatedLines = [];
  28. foreach ($lines as $line) {
  29. // Basic sanity check: Ensure line is not empty after trimming.
  30. $trimmedLine = trim($line);
  31. if (empty($trimmedLine)) {
  32. continue;
  33. }
  35. // Deduplication logic
  36. if (is_array($fields)) {
  37. $key = array_column($line, $fields); // Create a key based on specified fields
  38. } else {
  39. $key = $line; // Use the entire line as the key if no fields are specified
  40. }
  42. if (!isset($seen[$key])) {
  43. $seen[$key] = true;
  44. $deduplicatedLines[] = $line;
  45. }
  46. }
  48. if (file_put_contents($outputFile, implode("\n", $deduplicatedLines)) === false) {
  49. error_log("Failed to write to output file: " . $outputFile);
  50. return false;
  51. }
  53. return true;
  54. }

Add your comment