38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
|
"""
|
||
|
|
Utility functions for data conversion and JSON serialization.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import pandas as pd
|
||
|
|
import numpy as np
|
||
|
|
from typing import Any, List
|
||
|
|
|
||
|
|
|
||
|
|
def convert_to_native(value: Any) -> Any:
|
||
|
|
"""Convert numpy types to native Python types for JSON serialization."""
|
||
|
|
if pd.isna(value):
|
||
|
|
return None
|
||
|
|
if isinstance(value, (np.integer,)):
|
||
|
|
return int(value)
|
||
|
|
if isinstance(value, (np.floating,)):
|
||
|
|
if np.isnan(value) or np.isinf(value):
|
||
|
|
return None
|
||
|
|
return float(value)
|
||
|
|
if isinstance(value, np.ndarray):
|
||
|
|
return value.tolist()
|
||
|
|
if value == "SUPP" or value == "NE" or value == "NA" or value == "NP":
|
||
|
|
return None
|
||
|
|
return value
|
||
|
|
|
||
|
|
|
||
|
|
def clean_for_json(df: pd.DataFrame) -> List[dict]:
|
||
|
|
"""Convert DataFrame to list of dicts, replacing NaN/inf with None for JSON serialization."""
|
||
|
|
records = df.to_dict(orient="records")
|
||
|
|
cleaned = []
|
||
|
|
for record in records:
|
||
|
|
clean_record = {}
|
||
|
|
for key, value in record.items():
|
||
|
|
clean_record[key] = convert_to_native(value)
|
||
|
|
cleaned.append(clean_record)
|
||
|
|
return cleaned
|
||
|
|
|