54e4bc2e77
All checks were successful
Build and Push Docker Image / build-and-push (push) Successful in 1m7s
38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
"""
|
|
Utility functions for data conversion and JSON serialization.
|
|
"""
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
from typing import Any, List
|
|
|
|
|
|
def convert_to_native(value: Any) -> Any:
|
|
"""Convert numpy types to native Python types for JSON serialization."""
|
|
if pd.isna(value):
|
|
return None
|
|
if isinstance(value, (np.integer,)):
|
|
return int(value)
|
|
if isinstance(value, (np.floating,)):
|
|
if np.isnan(value) or np.isinf(value):
|
|
return None
|
|
return float(value)
|
|
if isinstance(value, np.ndarray):
|
|
return value.tolist()
|
|
if value == "SUPP" or value == "NE" or value == "NA" or value == "NP":
|
|
return None
|
|
return value
|
|
|
|
|
|
def clean_for_json(df: pd.DataFrame) -> List[dict]:
|
|
"""Convert DataFrame to list of dicts, replacing NaN/inf with None for JSON serialization."""
|
|
records = df.to_dict(orient="records")
|
|
cleaned = []
|
|
for record in records:
|
|
clean_record = {}
|
|
for key, value in record.items():
|
|
clean_record[key] = convert_to_native(value)
|
|
cleaned.append(clean_record)
|
|
return cleaned
|
|
|