From 2d368aacfaa789c03aec75aabbdae42905cfb5aa Mon Sep 17 00:00:00 2001 From: Greg Hale Date: Wed, 30 Oct 2024 21:52:31 -0700 Subject: [PATCH] Fix up python serialization and add ruby tests --- .../src/types/function_results.rs | 29 +++++++++++++------ engine/language_client_ruby/Gemfile.lock | 2 +- integ-tests/python/tests/test_functions.py | 3 +- integ-tests/ruby/test_functions.rb | 11 +++++++ integ-tests/typescript/test-report.html | 19 +++++++++++- 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/engine/language_client_python/src/types/function_results.rs b/engine/language_client_python/src/types/function_results.rs index 4e447d0a9..18e343b3c 100644 --- a/engine/language_client_python/src/types/function_results.rs +++ b/engine/language_client_python/src/types/function_results.rs @@ -71,7 +71,6 @@ fn pythonize_strict( cls_module: &Bound<'_, PyModule>, ) -> PyResult { let meta = parsed.meta().clone(); - dbg!(&parsed); let py_value_without_constraints = match parsed { BamlValueWithMeta::String(val, _) => PyResult::Ok(val.into_py(py)), BamlValueWithMeta::Int(val, _) => Ok(val.into_py(py)), @@ -138,7 +137,24 @@ fn pythonize_strict( let properties_dict = pyo3::types::PyDict::new_bound(py); for (key, value) in properties { - properties_dict.set_item(key, value)?; + // For each field, try to call pydantic's `model_dump` on the + // field. This is necessary in case the field is `Checked[_,_]`, + // because pydantic requires to parse such fields from json, + // rather than from a Python object. The python object is an + // untyped Dict, but python expects a typed `Checked`. + // By turning such values into `json`, we allow pydantic's + // parser to more flexibly accept input at its expected + // type. + // + // This has the side-effect of calling `model_dump` on all + // classes inheriting `BaseModel`, which probably incurs some + // performance penalty. So we should consider testing whether + // the field is a `Checked` before doing a `model_dump`. + let value_model = value.call_method0(py, "model_dump"); + match value_model { + Err(_) => { properties_dict.set_item(key, value)?; } + Ok(m) => {properties_dict.set_item(key, m)?;} + } } let class_type = match cls_module.getattr(class_name.as_str()) { @@ -151,8 +167,8 @@ fn pythonize_strict( Err(_) => return Ok(properties_dict.into()), }; + let instance = class_type.call_method("model_validate", (properties_dict.clone(),), None)?; - let instance = class_type.call_method("model_validate", (properties_dict,), None)?; Ok(instance.into()) } BamlValueWithMeta::Null(_) => Ok(py.None()), @@ -193,13 +209,8 @@ fn pythonize_strict( let class_checked_type: Bound<'_, PyAny> = class_checked_type_constructor .call_method1("__class_getitem__", (type_parameters_tuple,))?; - // panic!("About to call model_validate on {literal_check_names:?} {meta:?}"); - if meta.len() > 1 { - panic!("{meta:?}"); - } - // Validate the model with the constructed type - let checked_instance = class_checked_type.call_method("model_validate", (properties_dict,), None)?; + let checked_instance = class_checked_type.call_method("model_validate", (properties_dict.clone(),), None)?; Ok(checked_instance.into()) } diff --git a/engine/language_client_ruby/Gemfile.lock b/engine/language_client_ruby/Gemfile.lock index 8b8f71576..0156021ce 100644 --- a/engine/language_client_ruby/Gemfile.lock +++ b/engine/language_client_ruby/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - baml (0.63.0) + baml (0.64.0) GEM remote: https://rubygems.org/ diff --git a/integ-tests/python/tests/test_functions.py b/integ-tests/python/tests/test_functions.py index 793c81878..58b25b738 100644 --- a/integ-tests/python/tests/test_functions.py +++ b/integ-tests/python/tests/test_functions.py @@ -1396,4 +1396,5 @@ async def test_block_constraints(): @pytest.mark.asyncio async def test_nested_block_constraints(): ret = await b.MakeNestedBlockConstraint() - assert ret.nbc.checks["cross_field"].status == "failed" + print(ret) + assert ret.nbc.checks["cross_field"].status == "succeeded" diff --git a/integ-tests/ruby/test_functions.rb b/integ-tests/ruby/test_functions.rb index e4ea441c5..53c716585 100644 --- a/integ-tests/ruby/test_functions.rb +++ b/integ-tests/ruby/test_functions.rb @@ -319,4 +319,15 @@ res = b.PredictAge(name: "Greg") assert_equal res["certainty"].checks[:unreasonably_certain].status, "failed" end + + it "uses block_level constraints" do + res = b.MakeBlockConstraint() + assert_equal res.checks[:cross_field].status, "failed" + end + + it "uses nested_block_level constraints" do + res = b.MakeNestedBlockConstraint() + assert_equal res["nbc"].checks[:cross_field].status, "succeeded" + end + end diff --git a/integ-tests/typescript/test-report.html b/integ-tests/typescript/test-report.html index f8b4c6ef1..a30236d8b 100644 --- a/integ-tests/typescript/test-report.html +++ b/integ-tests/typescript/test-report.html @@ -257,4 +257,21 @@ font-size: 1rem; padding: 0 0.5rem; } -

Test Report

Started: 2024-10-30 15:56:11
Suites (1)
1 passed
0 failed
0 pending
Tests (61)
4 passed
0 failed
57 pending
Integ tests > should work for all inputs
single bool
pending
0s
Integ tests > should work for all inputs
single string list
pending
0s
Integ tests > should work for all inputs
return literal union
pending
0s
Integ tests > should work for all inputs
single class
pending
0s
Integ tests > should work for all inputs
multiple classes
pending
0s
Integ tests > should work for all inputs
single enum list
pending
0s
Integ tests > should work for all inputs
single float
pending
0s
Integ tests > should work for all inputs
single int
pending
0s
Integ tests > should work for all inputs
single literal int
pending
0s
Integ tests > should work for all inputs
single literal bool
pending
0s
Integ tests > should work for all inputs
single literal string
pending
0s
Integ tests > should work for all inputs
single class with literal prop
pending
0s
Integ tests > should work for all inputs
single class with literal union prop
pending
0s
Integ tests > should work for all inputs
single optional string
pending
0s
Integ tests > should work for all inputs
single map string to string
pending
0s
Integ tests > should work for all inputs
single map string to class
pending
0s
Integ tests > should work for all inputs
single map string to map
pending
0s
Integ tests
should work for all outputs
pending
0s
Integ tests
works with retries1
pending
0s
Integ tests
works with retries2
pending
0s
Integ tests
works with fallbacks
pending
0s
Integ tests
should work with image from url
pending
0s
Integ tests
should work with image from base 64
pending
0s
Integ tests
should work with audio base 64
pending
0s
Integ tests
should work with audio from url
pending
0s
Integ tests
should support streaming in OpenAI
pending
0s
Integ tests
should support streaming in Gemini
pending
0s
Integ tests
should support AWS
pending
0s
Integ tests
should support streaming in AWS
pending
0s
Integ tests
should support OpenAI shorthand
pending
0s
Integ tests
should support OpenAI shorthand streaming
pending
0s
Integ tests
should support anthropic shorthand
pending
0s
Integ tests
should support anthropic shorthand streaming
pending
0s
Integ tests
should support streaming without iterating
pending
0s
Integ tests
should support streaming in Claude
pending
0s
Integ tests
should support vertex
pending
0s
Integ tests
supports tracing sync
pending
0s
Integ tests
supports tracing async
pending
0s
Integ tests
should work with dynamic types single
pending
0s
Integ tests
should work with dynamic types enum
pending
0s
Integ tests
should work with dynamic literals
pending
0s
Integ tests
should work with dynamic types class
pending
0s
Integ tests
should work with dynamic inputs class
pending
0s
Integ tests
should work with dynamic inputs list
pending
0s
Integ tests
should work with dynamic output map
pending
0s
Integ tests
should work with dynamic output union
pending
0s
Integ tests
should work with nested classes
pending
0s
Integ tests
should work with dynamic client
pending
0s
Integ tests
should work with 'onLogEvent'
pending
0s
Integ tests
should work with a sync client
pending
0s
Integ tests
should raise an error when appropriate
pending
0s
Integ tests
should raise a BAMLValidationError
pending
0s
Integ tests
should reset environment variables correctly
pending
0s
Integ tests
should use aliases when serializing input objects - classes
pending
0s
Integ tests
should use aliases when serializing, but still have original keys in jinja
pending
0s
Integ tests
should use aliases when serializing input objects - enums
pending
0s
Integ tests
should use aliases when serializing input objects - lists
pending
0s
Integ tests
constraints: should handle checks in return types
passed
0.769s
Integ tests
constraints: should handle checks in returned unions
passed
0.787s
Integ tests
constraints: should handle block-level checks
passed
0.746s
Integ tests
constraints: should handle nested-block-level checks
passed
0.656s
\ No newline at end of file +

Test Report

Started: 2024-10-30 21:43:04
Suites (1)
0 passed
1 failed
0 pending
Tests (61)
60 passed
1 failed
0 pending
Integ tests > should work for all inputs
single bool
passed
0.788s
Integ tests > should work for all inputs
single string list
passed
0.508s
Integ tests > should work for all inputs
return literal union
passed
0.492s
Integ tests > should work for all inputs
single class
passed
0.515s
Integ tests > should work for all inputs
multiple classes
passed
0.482s
Integ tests > should work for all inputs
single enum list
passed
0.64s
Integ tests > should work for all inputs
single float
passed
0.424s
Integ tests > should work for all inputs
single int
passed
0.405s
Integ tests > should work for all inputs
single literal int
passed
0.564s
Integ tests > should work for all inputs
single literal bool
passed
0.559s
Integ tests > should work for all inputs
single literal string
passed
0.532s
Integ tests > should work for all inputs
single class with literal prop
passed
0.535s
Integ tests > should work for all inputs
single class with literal union prop
passed
1.183s
Integ tests > should work for all inputs
single optional string
passed
0.614s
Integ tests > should work for all inputs
single map string to string
passed
0.902s
Integ tests > should work for all inputs
single map string to class
passed
0.834s
Integ tests > should work for all inputs
single map string to map
passed
0.755s
Integ tests
should work for all outputs
passed
6.728s
Integ tests
works with retries1
passed
1.379s
Integ tests
works with retries2
passed
2.844s
Integ tests
works with fallbacks
passed
1.885s
Integ tests
should work with image from url
passed
4.717s
Integ tests
should work with image from base 64
passed
1.653s
Integ tests
should work with audio base 64
passed
1.224s
Integ tests
should work with audio from url
passed
1.339s
Integ tests
should support streaming in OpenAI
passed
2.66s
Integ tests
should support streaming in Gemini
passed
7.581s
Integ tests
should support AWS
passed
2.559s
Integ tests
should support streaming in AWS
passed
2.138s
Integ tests
should support OpenAI shorthand
passed
7.385s
Integ tests
should support OpenAI shorthand streaming
passed
5.362s
Integ tests
should support anthropic shorthand
passed
2.93s
Integ tests
should support anthropic shorthand streaming
passed
2.608s
Integ tests
should support streaming without iterating
passed
3.037s
Integ tests
should support streaming in Claude
passed
1.315s
Integ tests
should support vertex
passed
8.709s
Integ tests
supports tracing sync
passed
0.016s
Integ tests
supports tracing async
passed
2.962s
Integ tests
should work with dynamic types single
passed
0.901s
Integ tests
should work with dynamic types enum
passed
0.882s
Integ tests
should work with dynamic literals
passed
1.278s
Integ tests
should work with dynamic types class
passed
1.635s
Integ tests
should work with dynamic inputs class
passed
0.714s
Integ tests
should work with dynamic inputs list
passed
0.568s
Integ tests
should work with dynamic output map
passed
0.861s
Integ tests
should work with dynamic output union
passed
2.259s
Integ tests
should work with nested classes
failed
5.494s
Error: expect(received).toEqual(expected) // deep equality
+
+- Expected  - 1
++ Received  + 1
+
+  Object {
+    "prop1": "value1",
+    "prop2": Object {
+      "inner": Object {
+        "prop2": 42,
+-       "prop3": 3.14,
++       "prop3": null,
+      },
+      "prop1": "value2",
+      "prop2": "value3",
+    },
+  }
+    at Object.toEqual (/Users/greghale/code/baml/integ-tests/typescript/tests/integ-tests.test.ts:580:25)
Integ tests
should work with dynamic client
passed
0.398s
Integ tests
should work with 'onLogEvent'
passed
2.203s
Integ tests
should work with a sync client
passed
0.594s
Integ tests
should raise an error when appropriate
passed
1.021s
Integ tests
should raise a BAMLValidationError
passed
0.625s
Integ tests
should reset environment variables correctly
passed
1.049s
Integ tests
should use aliases when serializing input objects - classes
passed
2.89s
Integ tests
should use aliases when serializing, but still have original keys in jinja
passed
1.174s
Integ tests
should use aliases when serializing input objects - enums
passed
0.503s
Integ tests
should use aliases when serializing input objects - lists
passed
0.428s
Integ tests
constraints: should handle checks in return types
passed
0.863s
Integ tests
constraints: should handle checks in returned unions
passed
0.866s
Integ tests
constraints: should handle block-level checks
passed
0.631s
Integ tests
constraints: should handle nested-block-level checks
passed
0.761s
\ No newline at end of file