Skip to content

Fix to_json serialization for floats #191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 10, 2024
Merged

Fix to_json serialization for floats #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 43 additions & 3 deletions cpp/csp/python/PyStructToJson.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,32 @@ inline rapidjson::Value toJson( const T& val, const CspType& typ, rapidjson::Doc
return rapidjson::Value( val );
}

// Helper function for parsing doubles
inline rapidjson::Value doubleToJson( const double& val, rapidjson::Document& doc )
{
// NOTE: Rapidjson adds support for this in a future release. Remove this when we upgrade rapidjson to a version
// after 07/16/2023 and use kWriteNanAndInfNullFlag in the writer.
//
// To be compatible with other JSON libraries, we cannot use the default approach that rapidjson has to
// serializing NaN, and (+/-)Infs. We need to manually convert them to NULLs. Rapidjson adds support for this
// in a future release.
if ( std::isnan( val ) || std::isinf( val ) )
{
return rapidjson::Value();
}
else
{
return rapidjson::Value( val );
}
}

// Helper function to convert doubles into json format recursively, by properly handlings NaNs, and Infs
template<>
inline rapidjson::Value toJson( const double& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable )
{
return doubleToJson( val, doc );
}

// Helper function to convert Enums into json format recursively
template<>
inline rapidjson::Value toJson( const CspEnum& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable )
Expand Down Expand Up @@ -183,7 +209,21 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc )
else if( PyFloat_Check( py_key ) )
{
auto key = PyFloat_AsDouble( py_key );
val.SetString( std::to_string( key ), doc.GetAllocator() );
auto json_obj = doubleToJson( key, doc );
if ( json_obj.IsNull() )
{
auto * str_obj = PyObject_Str( py_key );
Py_ssize_t len = 0;
const char * str = PyUnicode_AsUTF8AndSize( str_obj, &len );
CSP_THROW( ValueError, "Cannot serialize " + std::string( str ) + " to key in JSON" );
}
else
{
// Convert to string
std::stringstream s;
s << key;
val.SetString( s.str(), doc.GetAllocator() );
}
}
else
{
Expand Down Expand Up @@ -255,12 +295,12 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO
}
else if( PyFloat_Check( value ) )
{
return rapidjson::Value( fromPython<double>( value ) );
return doubleToJson( fromPython<double>( value ), doc );
}
else if( PyUnicode_Check( value ) )
{
Py_ssize_t len;
auto str = PyUnicode_AsUTF8AndSize( value , &len );
auto str = PyUnicode_AsUTF8AndSize( value, &len );
rapidjson::Value str_val;
str_val.SetString( str, len, doc.GetAllocator() );
return str_val;
Expand Down
52 changes: 50 additions & 2 deletions csp/tests/impl/test_struct.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1282,6 +1282,18 @@ class MyStruct(csp.Struct):
result_dict = {"b": False, "i": 456, "f": 1.73, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

test_struct = MyStruct(b=False, i=456, f=float("nan"), s="789")
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

test_struct = MyStruct(b=False, i=456, f=float("inf"), s="789")
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

test_struct = MyStruct(b=False, i=456, f=float("-inf"), s="789")
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

def test_to_json_enums(self):
from enum import Enum as PyEnum

Expand Down Expand Up @@ -1434,8 +1446,13 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "l_any": l_l_i}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [12, False]))]]]
l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [12, False]]]]]]
l_any = [[1, float("nan")], [float("INFINITY"), float("-inf")]]
test_struct = MyStruct(i=456, l_any=l_any)
result_dict = {"i": 456, "l_any": [[1, None], [None, None]]}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [float("nan"), False]))]]]
l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [None, False]]]]]]
test_struct = MyStruct(i=456, l_any=l_any)
result_dict = {"i": 456, "l_any": l_any_result}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
Expand All @@ -1444,6 +1461,7 @@ def test_to_json_dict(self):
class MyStruct(csp.Struct):
i: int = 123
d_i: typing.Dict[int, int]
d_f: typing.Dict[float, int]
d_dt: typing.Dict[str, datetime]
d_d_s: typing.Dict[str, typing.Dict[str, str]]
d_any: dict
Expand All @@ -1458,6 +1476,12 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "d_i": d_i_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_f=d_f)
result_dict = {"i": 456, "d_f": d_f_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

dt = datetime.now(tz=pytz.utc)
d_dt = {"d1": dt, "d2": dt}
test_struct = MyStruct(i=456, d_dt=d_dt)
Expand All @@ -1475,6 +1499,12 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "d_any": d_i_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
result_dict = {"i": 456, "d_any": d_f_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

dt = datetime.now(tz=pytz.utc)
d_dt = {"d1": dt, "d2": dt}
test_struct = MyStruct(i=456, d_any=d_dt)
Expand All @@ -1487,6 +1517,24 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "d_any": d_any_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

d_f = {float("nan"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
with self.assertRaises(ValueError):
test_struct.to_json()

d_f = {float("inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
with self.assertRaises(ValueError):
test_struct.to_json()

d_f = {float("-inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
with self.assertRaises(ValueError):
test_struct.to_json()

def test_to_json_struct(self):
class MySubSubStruct(csp.Struct):
b: bool = True
Expand Down