Skip to content

Commit 118a2fb

Browse files
arhamchopraCarreau
arhamchopra
authored and
Carreau
committed
Fix to_json serialization for floats
Signed-off-by: Arham Chopra <[email protected]>
1 parent c51a86b commit 118a2fb

File tree

2 files changed

+93
-5
lines changed

2 files changed

+93
-5
lines changed

‎cpp/csp/python/PyStructToJson.cpp‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,32 @@ inline rapidjson::Value toJson( const T& val, const CspType& typ, rapidjson::Doc
2121
return rapidjson::Value( val );
2222
}
2323

24+
// Helper function for parsing doubles
25+
inline rapidjson::Value doubleToJson( const double& val, rapidjson::Document& doc )
26+
{
27+
// NOTE: Rapidjson adds support for this in a future release. Remove this when we upgrade rapidjson to a version
28+
// after 07/16/2023 and use kWriteNanAndInfNullFlag in the writer.
29+
//
30+
// To be compatible with other JSON libraries, we cannot use the default approach that rapidjson has to
31+
// serializing NaN, and (+/-)Infs. We need to manually convert them to NULLs. Rapidjson adds support for this
32+
// in a future release.
33+
if ( std::isnan( val ) || std::isinf( val ) )
34+
{
35+
return rapidjson::Value();
36+
}
37+
else
38+
{
39+
return rapidjson::Value( val );
40+
}
41+
}
42+
43+
// Helper function to convert doubles into json format recursively, by properly handlings NaNs, and Infs
44+
template<>
45+
inline rapidjson::Value toJson( const double& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable )
46+
{
47+
return doubleToJson( val, doc );
48+
}
49+
2450
// Helper function to convert Enums into json format recursively
2551
template<>
2652
inline rapidjson::Value toJson( const CspEnum& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable )
@@ -183,7 +209,21 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc )
183209
else if( PyFloat_Check( py_key ) )
184210
{
185211
auto key = PyFloat_AsDouble( py_key );
186-
val.SetString( std::to_string( key ), doc.GetAllocator() );
212+
auto json_obj = doubleToJson( key, doc );
213+
if ( json_obj.IsNull() )
214+
{
215+
auto * str_obj = PyObject_Str( py_key );
216+
Py_ssize_t len = 0;
217+
const char * str = PyUnicode_AsUTF8AndSize( str_obj, &len );
218+
CSP_THROW( ValueError, "Cannot serialize " + std::string( str ) + " to key in JSON" );
219+
}
220+
else
221+
{
222+
// Convert to string
223+
std::stringstream s;
224+
s << key;
225+
val.SetString( s.str(), doc.GetAllocator() );
226+
}
187227
}
188228
else
189229
{
@@ -255,12 +295,12 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO
255295
}
256296
else if( PyFloat_Check( value ) )
257297
{
258-
return rapidjson::Value( fromPython<double>( value ) );
298+
return doubleToJson( fromPython<double>( value ), doc );
259299
}
260300
else if( PyUnicode_Check( value ) )
261301
{
262302
Py_ssize_t len;
263-
auto str = PyUnicode_AsUTF8AndSize( value , &len );
303+
auto str = PyUnicode_AsUTF8AndSize( value, &len );
264304
rapidjson::Value str_val;
265305
str_val.SetString( str, len, doc.GetAllocator() );
266306
return str_val;

‎csp/tests/impl/test_struct.py‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1282,6 +1282,18 @@ class MyStruct(csp.Struct):
12821282
result_dict = {"b": False, "i": 456, "f": 1.73, "s": "789"}
12831283
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
12841284

1285+
test_struct = MyStruct(b=False, i=456, f=float("nan"), s="789")
1286+
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
1287+
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
1288+
1289+
test_struct = MyStruct(b=False, i=456, f=float("inf"), s="789")
1290+
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
1291+
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
1292+
1293+
test_struct = MyStruct(b=False, i=456, f=float("-inf"), s="789")
1294+
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
1295+
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
1296+
12851297
def test_to_json_enums(self):
12861298
from enum import Enum as PyEnum
12871299

@@ -1434,8 +1446,13 @@ class MyStruct(csp.Struct):
14341446
result_dict = {"i": 456, "l_any": l_l_i}
14351447
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
14361448

1437-
l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [12, False]))]]]
1438-
l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [12, False]]]]]]
1449+
l_any = [[1, float("nan")], [float("INFINITY"), float("-inf")]]
1450+
test_struct = MyStruct(i=456, l_any=l_any)
1451+
result_dict = {"i": 456, "l_any": [[1, None], [None, None]]}
1452+
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
1453+
1454+
l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [float("nan"), False]))]]]
1455+
l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [None, False]]]]]]
14391456
test_struct = MyStruct(i=456, l_any=l_any)
14401457
result_dict = {"i": 456, "l_any": l_any_result}
14411458
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
@@ -1444,6 +1461,7 @@ def test_to_json_dict(self):
14441461
class MyStruct(csp.Struct):
14451462
i: int = 123
14461463
d_i: typing.Dict[int, int]
1464+
d_f: typing.Dict[float, int]
14471465
d_dt: typing.Dict[str, datetime]
14481466
d_d_s: typing.Dict[str, typing.Dict[str, str]]
14491467
d_any: dict
@@ -1458,6 +1476,12 @@ class MyStruct(csp.Struct):
14581476
result_dict = {"i": 456, "d_i": d_i_res}
14591477
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
14601478

1479+
d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7}
1480+
d_f_res = {str(k): v for k, v in d_f.items()}
1481+
test_struct = MyStruct(i=456, d_f=d_f)
1482+
result_dict = {"i": 456, "d_f": d_f_res}
1483+
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
1484+
14611485
dt = datetime.now(tz=pytz.utc)
14621486
d_dt = {"d1": dt, "d2": dt}
14631487
test_struct = MyStruct(i=456, d_dt=d_dt)
@@ -1475,6 +1499,12 @@ class MyStruct(csp.Struct):
14751499
result_dict = {"i": 456, "d_any": d_i_res}
14761500
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
14771501

1502+
d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7}
1503+
d_f_res = {str(k): v for k, v in d_f.items()}
1504+
test_struct = MyStruct(i=456, d_any=d_f)
1505+
result_dict = {"i": 456, "d_any": d_f_res}
1506+
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
1507+
14781508
dt = datetime.now(tz=pytz.utc)
14791509
d_dt = {"d1": dt, "d2": dt}
14801510
test_struct = MyStruct(i=456, d_any=d_dt)
@@ -1487,6 +1517,24 @@ class MyStruct(csp.Struct):
14871517
result_dict = {"i": 456, "d_any": d_any_res}
14881518
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
14891519

1520+
d_f = {float("nan"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
1521+
d_f_res = {str(k): v for k, v in d_f.items()}
1522+
test_struct = MyStruct(i=456, d_any=d_f)
1523+
with self.assertRaises(ValueError):
1524+
test_struct.to_json()
1525+
1526+
d_f = {float("inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
1527+
d_f_res = {str(k): v for k, v in d_f.items()}
1528+
test_struct = MyStruct(i=456, d_any=d_f)
1529+
with self.assertRaises(ValueError):
1530+
test_struct.to_json()
1531+
1532+
d_f = {float("-inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
1533+
d_f_res = {str(k): v for k, v in d_f.items()}
1534+
test_struct = MyStruct(i=456, d_any=d_f)
1535+
with self.assertRaises(ValueError):
1536+
test_struct.to_json()
1537+
14901538
def test_to_json_struct(self):
14911539
class MySubSubStruct(csp.Struct):
14921540
b: bool = True

0 commit comments

Comments
 (0)