1. import json
  2. import heapq
  4. def sort_json_records(json_file_path, key_path, sort_order='asc', chunk_size=1000):
  5. """
  6. Sorts records of JSON objects from a file with limited memory usage.
  8. Args:
  9. json_file_path (str): Path to the JSON file.
  10. key_path (str): Path to the key used for sorting (e.g., "field1.field2").
  11. sort_order (str): 'asc' for ascending, 'desc' for descending.
  12. chunk_size (int): Number of records to process in each chunk.
  14. Returns:
  15. list: Sorted list of JSON objects.
  16. """
  18. results = []
  19. with open(json_file_path, 'r') as f:
  20. for chunk in read_in_chunks(f, chunk_size):
  21. for record in chunk:
  22. try:
  23. record_dict = json.loads(record)
  24. value = get_value_from_path(record_dict, key_path)
  26. if sort_order == 'asc':
  27. heapq.heappush(results, (value, record_dict))
  28. elif sort_order == 'desc':
  29. heapq.heappush(results, (-value, record_dict)) #Negate for descending order
  30. else:
  31. raise ValueError("Invalid sort_order. Use 'asc' or 'desc'.")
  32. except (json.JSONDecodeError, KeyError, TypeError) as e:
  33. print(f"Skipping invalid record: {record}. Error: {e}")
  34. continue
  36. # Extract sorted records from the heap
  37. sorted_records = [record for _, record in heapq.nlargest(len(results), results)]
  39. return sorted_records
  43. def read_in_chunks(file_object, chunk_size):
  44. """
  45. Helper function to read a file in chunks.
  46. """
  47. while True:
  48. chunk = []
  49. for _ in range(chunk_size):
  50. line = file_object.readline()
  51. if not line:
  52. break
  53. chunk.append(line)
  54. if not chunk:
  55. break
  56. yield chunk
  58. def get_value_from_path(data, key_path):
  59. """
  60. Helper function to get a value from a nested dictionary using a path.
  61. """
  62. keys = key_path.split('.')
  63. value = data
  64. for key in keys:
  65. if isinstance(value, dict) and key in value:
  66. value = value[key]
  67. else:
  68. raise KeyError(f"Key '{key}' not found in data.")
  69. return value

Add your comment