Change RowAsDictJsonCoder to not ensure ASCII while encoding

Signed-off-by: Seunghwan Hong <[email protected]>
apache · Jul 17, 2022 · c390899 · c390899
1 parent 20274f3
commit c390899
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
@@ -1537,7 +1537,7 @@ def encode(self, table_row):
     # to the programmer that they have used NAN/INF values.
     try:
       return json.dumps(
-          table_row, allow_nan=False, default=default_encoder).encode('utf-8')
+          table_row, allow_nan=False, ensure_ascii=False, default=default_encoder).encode('utf-8')
     except ValueError as e:
       raise ValueError(
           '%s. %s. Row: %r' % (e, JSON_COMPLIANCE_ERROR, table_row))

diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
@@ -1065,6 +1065,13 @@ def test_invalid_json_inf(self):
   def test_invalid_json_neg_inf(self):
     self.json_compliance_exception(float('-inf'))
 
+  def test_ensure_ascii(self):
+    coder = RowAsDictJsonCoder()
+    test_value = {'s': '🎉'}
+    should_be = b'{"s": "\xf0\x9f\x8e\x89"}'
+
+    output_value = coder.encode(test_value)
+    self.assertEqual(output_value, should_be)
 
 @unittest.skipIf(HttpError is None, 'GCP dependencies are not installed')
 class TestJsonRowWriter(unittest.TestCase):