1. /**
  2. * Formats dataset output with rate limiting.
  3. *
  4. * @param {Array} data The dataset to format.
  5. * @param {number} rateLimit The maximum number of outputs per time unit (e.g., 100).
  6. * @param {number} timeUnit The time unit for the rate limit (e.g., 1000 for milliseconds).
  7. * @param {function} formatItem A function to format each item in the dataset. Should return a string.
  8. * @returns {Promise<Array>} A promise that resolves to an array of formatted strings.
  9. */
  10. async function formatDatasetWithRateLimit(data, rateLimit, timeUnit, formatItem) {
  11. let processed = 0;
  12. const queue = [...data]; // Create a copy to avoid modifying the original
  13. let lastProcessed = 0;
  15. async function processNext() {
  16. if (queue.length === 0) {
  17. return;
  18. }
  20. const item = queue.shift();
  21. const now = Date.now();
  23. if (now - lastProcessed >= timeUnit) {
  24. lastProcessed = now;
  25. }
  27. try {
  28. const formattedItem = formatItem(item);
  29. console.log(formattedItem); // Simulate output
  30. processed++;
  32. if (processed >= rateLimit) {
  33. await new Promise(resolve => setTimeout(resolve, timeUnit - (now - lastProcessed))); // Wait
  34. lastProcessed = Date.now();
  35. }
  37. await processNext(); // Process the next item
  38. } catch (error) {
  39. console.error("Error formatting item:", item, error);
  40. // Handle the error appropriately, e.g., log it or skip the item.
  41. // Consider adding retry logic.
  42. }
  43. }
  45. // Start processing the queue
  46. await processNext();
  48. return queue; // Return any remaining items in the queue
  49. }
  52. //Example usage:
  53. // const myData = [ {id:1, name:"Item 1"}, {id:2, name:"Item 2"}, {id:3, name:"Item 3"}, {id:4, name:"Item 4"}, {id:5, name:"Item 5"}];
  54. // formatDatasetWithRateLimit(myData, 50, 1000, (item) => `Item ${item.id}: ${item.name}`);

Add your comment