From e26f8ab4416f8cdf91171a6e9143785b78863ade Mon Sep 17 00:00:00 2001 From: Dave Cunningham Date: Thu, 2 Jun 2016 01:20:10 -0400 Subject: [PATCH 1/3] Return arbitrary json from native extensions --- core/json.h | 10 ++-- core/libjsonnet.cpp | 31 ++++++++++++ core/libjsonnet_test_snippet.c | 42 ++++++++++++++-- core/state.h | 5 +- core/vm.cpp | 90 ++++++++++++++++++++++++---------- include/libjsonnet.h | 27 ++++++++++ 6 files changed, 173 insertions(+), 32 deletions(-) diff --git a/core/json.h b/core/json.h index e99d310da..d1c546308 100644 --- a/core/json.h +++ b/core/json.h @@ -17,22 +17,26 @@ limitations under the License. #ifndef JSONNET_JSON_H #define JSONNET_JSON_H +#include +#include #include #include struct JsonnetJsonValue { enum Kind { - ARRAY, // Not implemented yet. + ARRAY, BOOL, NULL_KIND, NUMBER, - OBJECT, // Not implemented yet. + OBJECT, STRING, }; Kind kind; std::string string; - double number; // Also used for bool (0 and 1) + double number; // Also used for bool (0.0 and 1.0) + std::vector> elements; + std::map> fields; }; #endif diff --git a/core/libjsonnet.cpp b/core/libjsonnet.cpp index 972c0402d..ea29570ba 100644 --- a/core/libjsonnet.cpp +++ b/core/libjsonnet.cpp @@ -116,6 +116,37 @@ JsonnetJsonValue *jsonnet_json_make_null(struct JsonnetVm *vm) return r; } +JsonnetJsonValue *jsonnet_json_make_array(JsonnetVm *vm) +{ + (void) vm; + JsonnetJsonValue *r = new JsonnetJsonValue(); + r->kind = JsonnetJsonValue::ARRAY; + return r; +} + +void jsonnet_json_array_append(JsonnetVm *vm, JsonnetJsonValue *arr, JsonnetJsonValue *v) +{ + (void) vm; + assert(arr->kind == JsonnetJsonValue::ARRAY); + arr->elements.emplace_back(v); +} + +JsonnetJsonValue *jsonnet_json_make_object(JsonnetVm *vm) +{ + (void) vm; + JsonnetJsonValue *r = new JsonnetJsonValue(); + r->kind = JsonnetJsonValue::OBJECT; + return r; +} + +void jsonnet_json_object_append(JsonnetVm *vm, JsonnetJsonValue *obj, + const char *f, JsonnetJsonValue *v) +{ + (void) vm; + assert(obj->kind == JsonnetJsonValue::OBJECT); + obj->fields[std::string(f)] = std::unique_ptr(v); +} + struct JsonnetVm { double gcGrowthTrigger; unsigned maxStack; diff --git a/core/libjsonnet_test_snippet.c b/core/libjsonnet_test_snippet.c index ba9e989e7..c8caac3f9 100644 --- a/core/libjsonnet_test_snippet.c +++ b/core/libjsonnet_test_snippet.c @@ -21,7 +21,9 @@ limitations under the License. #include -struct JsonnetJsonValue *native_concat(void *ctx, const struct JsonnetJsonValue * const *argv, int *succ) +typedef struct JsonnetJsonValue JJV; + +static JJV *native_concat(void *ctx, const JJV * const *argv, int *succ) { struct JsonnetVm *vm = (struct JsonnetVm *)ctx; const char *a = jsonnet_json_extract_string(vm, argv[0]); @@ -32,13 +34,13 @@ struct JsonnetJsonValue *native_concat(void *ctx, const struct JsonnetJsonValue } char *str = malloc(strlen(a) + strlen(b) + 1); sprintf(str, "%s%s", a, b); - struct JsonnetJsonValue *r = jsonnet_json_make_string(vm, str); + JJV *r = jsonnet_json_make_string(vm, str); free(str); *succ = 1; return r; } -struct JsonnetJsonValue *native_square(void *ctx, const struct JsonnetJsonValue * const *argv, int *succ) +static JJV *native_square(void *ctx, const JJV * const *argv, int *succ) { struct JsonnetVm *vm = (struct JsonnetVm *)ctx; double a; @@ -50,11 +52,44 @@ struct JsonnetJsonValue *native_square(void *ctx, const struct JsonnetJsonValue return jsonnet_json_make_number(vm, a * a); } +static JJV *native_build(void *ctx, const JJV * const *argv, int *succ) +{ + struct JsonnetVm *vm = (struct JsonnetVm *)ctx; + (void) argv; + JJV *obj_top = jsonnet_json_make_object(vm); + JJV *arr_top = jsonnet_json_make_array(vm); + JJV *arr1 = jsonnet_json_make_array(vm); + jsonnet_json_array_append(vm, arr1, jsonnet_json_make_string(vm, "Test 1.1")); + jsonnet_json_array_append(vm, arr1, jsonnet_json_make_string(vm, "Test 1.2")); + jsonnet_json_array_append(vm, arr1, jsonnet_json_make_string(vm, "Test 1.3")); + jsonnet_json_array_append(vm, arr1, jsonnet_json_make_bool(vm, 1)); + jsonnet_json_array_append(vm, arr1, jsonnet_json_make_number(vm, 42)); + jsonnet_json_array_append(vm, arr1, jsonnet_json_make_null(vm)); + jsonnet_json_array_append(vm, arr1, jsonnet_json_make_object(vm)); + jsonnet_json_array_append(vm, arr_top, arr1); + JJV *arr2 = jsonnet_json_make_array(vm); + jsonnet_json_array_append(vm, arr2, jsonnet_json_make_string(vm, "Test 2.1")); + jsonnet_json_array_append(vm, arr2, jsonnet_json_make_string(vm, "Test 2.2")); + jsonnet_json_array_append(vm, arr2, jsonnet_json_make_string(vm, "Test 2.3")); + jsonnet_json_array_append(vm, arr2, jsonnet_json_make_bool(vm, 0)); + jsonnet_json_array_append(vm, arr2, jsonnet_json_make_number(vm, -42)); + jsonnet_json_array_append(vm, arr2, jsonnet_json_make_null(vm)); + JJV *little_obj = jsonnet_json_make_object(vm); + jsonnet_json_object_append(vm, little_obj, "f", jsonnet_json_make_string(vm, "foo")); + jsonnet_json_object_append(vm, little_obj, "g", jsonnet_json_make_string(vm, "bar")); + jsonnet_json_array_append(vm, arr2, little_obj); + jsonnet_json_array_append(vm, arr_top, arr2); + jsonnet_json_object_append(vm, obj_top, "field", arr_top); + *succ = 1; + return obj_top; +} + int main(int argc, const char **argv) { int error; char *output; struct JsonnetVm *vm; + const char *params0[] = {NULL}; const char *params1[] = {"a", NULL}; const char *params2[] = {"a", "b", NULL}; if (argc != 2) { @@ -64,6 +99,7 @@ int main(int argc, const char **argv) vm = jsonnet_make(); jsonnet_native_callback(vm, "concat", native_concat, vm, params2); jsonnet_native_callback(vm, "square", native_square, vm, params1); + jsonnet_native_callback(vm, "build", native_build, vm, params0); output = jsonnet_evaluate_snippet(vm, "snippet", argv[1], &error); if (error) { fprintf(stderr, "%s", output); diff --git a/core/state.h b/core/state.h index 51ab9371a..e053512b2 100644 --- a/core/state.h +++ b/core/state.h @@ -209,8 +209,11 @@ struct HeapComprehensionObject : public HeapLeafObject { * For each field, holds the value that should be bound to id. This is the corresponding * array element from the original array used to define this object. This should not really * be a thunk, but it makes the implementation easier. + * + * It is convenient to make this non-const to allow building up the values one by one, so that + * the garbage collector can see them at each intermediate point. */ - const std::map compValues; + std::map compValues; HeapComprehensionObject(const BindingFrame &up_values, const AST *value, const Identifier *id, diff --git a/core/vm.cpp b/core/vm.cpp index d823ded1c..255674a73 100644 --- a/core/vm.cpp +++ b/core/vm.cpp @@ -451,6 +451,12 @@ class Interpreter { /** Used to "name" thunks created to execute invariants. */ const Identifier *idInvariant; + /** Used to "name" thunks created to convert JSON to Jsonnet objects. */ + const Identifier *idJsonObjVar; + + /** Used to refer to idJsonObjVar. */ + const AST *jsonObjVar; + struct ImportCacheValue { std::string foundHere; std::string content; @@ -779,6 +785,8 @@ class Interpreter { alloc(alloc), idArrayElement(alloc->makeIdentifier(U"array_element")), idInvariant(alloc->makeIdentifier(U"object_assert")), + idJsonObjVar(alloc->makeIdentifier(U"_")), + jsonObjVar(alloc->make(LocationRange(), Fodder{}, idJsonObjVar)), externalVars(ext_vars), nativeCallbacks(native_callbacks), importCallback(import_callback), @@ -1262,6 +1270,52 @@ class Interpreter { return nullptr; } + void jsonToHeap(const std::unique_ptr &v, Value &attach) + { + // In order to not anger the garbage collector, assign to attach immediately after + // making the heap object. + switch (v->kind) { + case JsonnetJsonValue::STRING: + attach = makeString(decode_utf8(v->string)); + break; + + case JsonnetJsonValue::BOOL: + attach = makeBoolean(v->number != 0.0); + break; + + case JsonnetJsonValue::NUMBER: + attach = makeDouble(v->number); + break; + + case JsonnetJsonValue::NULL_KIND: + attach = makeNull(); + break; + + case JsonnetJsonValue::ARRAY: { + attach = makeArray(std::vector{}); + auto *arr = static_cast(attach.v.h); + for (size_t i = 0; i < v->elements.size() ; ++i) { + arr->elements.push_back( + makeHeap(idArrayElement, nullptr, 0, nullptr)); + arr->elements[i]->filled = true; + jsonToHeap(v->elements[i], arr->elements[i]->content); + } + } break; + + case JsonnetJsonValue::OBJECT: { + attach = makeObject( + BindingFrame{}, jsonObjVar, idJsonObjVar, BindingFrame{}); + auto *obj = static_cast(attach.v.h); + for (const auto &pair : v->fields) { + auto *thunk = makeHeap(idJsonObjVar, nullptr, 0, nullptr); + obj->compValues[alloc->makeIdentifier(decode_utf8(pair.first))] = thunk; + thunk->filled = true; + jsonToHeap(pair.second, thunk->content); + } + } break; + } + } + String toString(const LocationRange &loc) { @@ -1983,6 +2037,8 @@ class Interpreter { JsonnetJsonValue::STRING, encode_utf8(static_cast(arg.v.h)->value), 0, + {}, + {}, }); break; @@ -1990,7 +2046,9 @@ class Interpreter { args2.push_back(JsonnetJsonValue{ JsonnetJsonValue::BOOL, "", - arg.v.b ? 0.0 : 1.0, + arg.v.b ? 1.0 : 0.0, + {}, + {}, }); break; @@ -1999,6 +2057,8 @@ class Interpreter { JsonnetJsonValue::NUMBER, "", arg.v.d, + {}, + {}, }); break; @@ -2007,6 +2067,8 @@ class Interpreter { JsonnetJsonValue::NULL_KIND, "", 0, + {}, + {}, }); break; @@ -2028,31 +2090,9 @@ class Interpreter { int succ; std::unique_ptr r(cb.cb(cb.ctx, &args3[0], &succ)); - if (succ) { - // TODO(dcunnin): Support arrays. - // TODO(dcunnin): Support objects. - switch (r->kind) { - case JsonnetJsonValue::STRING: - scratch = makeString(decode_utf8(r->string)); - break; - - case JsonnetJsonValue::BOOL: - scratch = makeBoolean(r->number != 0.0); - break; - - case JsonnetJsonValue::NUMBER: - scratch = makeDouble(r->number); - break; - - case JsonnetJsonValue::NULL_KIND: - scratch = makeNull(); - break; - - default: - throw makeError(ast.location, - "Native extensions can only return primitives."); - } + if (succ) { + jsonToHeap(r, scratch); } else { if (r->kind != JsonnetJsonValue::STRING) { throw makeError( diff --git a/include/libjsonnet.h b/include/libjsonnet.h index 978d2fb4f..da7e70754 100644 --- a/include/libjsonnet.h +++ b/include/libjsonnet.h @@ -104,6 +104,33 @@ struct JsonnetJsonValue *jsonnet_json_make_bool(struct JsonnetVm *vm, int v); */ struct JsonnetJsonValue *jsonnet_json_make_null(struct JsonnetVm *vm); +/** Make a JsonnetJsonValue representing an array. + * + * Assign elements with jsonnet_json_array_append. + */ +struct JsonnetJsonValue *jsonnet_json_make_array(struct JsonnetVm *vm); + +/** Add v to the end of the array. + */ +void jsonnet_json_array_append(struct JsonnetVm *vm, + struct JsonnetJsonValue *arr, + struct JsonnetJsonValue *v); + +/** Make a JsonnetJsonValue representing an object with the given number of fields. + * + * Every index of the array must have a unique value assigned with jsonnet_json_array_element. + */ +struct JsonnetJsonValue *jsonnet_json_make_object(struct JsonnetVm *vm); + +/** Add the field f to the object, bound to v. + * + * This replaces any previous binding of the field. + */ +void jsonnet_json_object_append(struct JsonnetVm *vm, + struct JsonnetJsonValue *obj, + const char *f, + struct JsonnetJsonValue *v); + /** Callback to provide native extensions to Jsonnet. * * The returned JsonnetJsonValue* should be allocated with jsonnet_realloc. It will be cleaned up From a2a01f7d96d5e2a664b2bae904bd94f218dc79da Mon Sep 17 00:00:00 2001 From: Dave Cunningham Date: Thu, 2 Jun 2016 20:31:50 -0400 Subject: [PATCH 2/3] Native python extensions can return arbitrary JSON --- core/libjsonnet.cpp | 6 +++ include/libjsonnet.h | 6 +++ python/_jsonnet.c | 93 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/core/libjsonnet.cpp b/core/libjsonnet.cpp index ea29570ba..ec759edc7 100644 --- a/core/libjsonnet.cpp +++ b/core/libjsonnet.cpp @@ -147,6 +147,12 @@ void jsonnet_json_object_append(JsonnetVm *vm, JsonnetJsonValue *obj, obj->fields[std::string(f)] = std::unique_ptr(v); } +void jsonnet_json_destroy(JsonnetVm *vm, JsonnetJsonValue *v) +{ + (void) vm; + delete v; +} + struct JsonnetVm { double gcGrowthTrigger; unsigned maxStack; diff --git a/include/libjsonnet.h b/include/libjsonnet.h index da7e70754..6848df379 100644 --- a/include/libjsonnet.h +++ b/include/libjsonnet.h @@ -131,6 +131,12 @@ void jsonnet_json_object_append(struct JsonnetVm *vm, const char *f, struct JsonnetJsonValue *v); +/** Clean up a JSON subtree. + * + * This is useful if you want to abort with an error mid-way through building a complex value. + */ +void jsonnet_json_destroy(struct JsonnetVm *vm, struct JsonnetJsonValue *v); + /** Callback to provide native extensions to Jsonnet. * * The returned JsonnetJsonValue* should be allocated with jsonnet_realloc. It will be cleaned up diff --git a/python/_jsonnet.c b/python/_jsonnet.c index e6ef75d05..491e1fb06 100644 --- a/python/_jsonnet.c +++ b/python/_jsonnet.c @@ -42,6 +42,72 @@ struct NativeCtx { size_t argc; }; +static struct JsonnetJsonValue *python_to_jsonnet_json(struct JsonnetVm *vm, PyObject *v, + const char **err_msg) +{ + if (PyString_Check(v)) { + return jsonnet_json_make_string(vm, PyString_AsString(v)); + } else if (PyUnicode_Check(v)) { + struct JsonnetJsonValue *r; + PyObject *str = PyUnicode_AsUTF8String(v); + r = jsonnet_json_make_string(vm, PyString_AsString(str)); + Py_DECREF(str); + return r; + } else if (PyFloat_Check(v)) { + return jsonnet_json_make_number(vm, PyFloat_AsDouble(v)); + } else if (PyInt_Check(v)) { + return jsonnet_json_make_number(vm, (double)(PyInt_AsLong(v))); + } else if (PyBool_Check(v)) { + return jsonnet_json_make_bool(vm, PyObject_IsTrue(v)); + } else if (v == Py_None) { + return jsonnet_json_make_null(vm); + } else if (PySequence_Check(v)) { + Py_ssize_t len, i; + struct JsonnetJsonValue *arr; + // Convert it to a O(1) indexable form if necessary. + PyObject *fast = PySequence_Fast(v, "python_to_jsonnet_json internal error: not sequence"); + len = PySequence_Fast_GET_SIZE(fast); + arr = jsonnet_json_make_array(vm); + for (i = 0; i < len; ++i) { + struct JsonnetJsonValue *json_el; + PyObject *el = PySequence_Fast_GET_ITEM(fast, i); + json_el = python_to_jsonnet_json(vm, el, err_msg); + if (json_el == NULL) { + Py_DECREF(fast); + jsonnet_json_destroy(vm, arr); + return NULL; + } + jsonnet_json_array_append(vm, arr, json_el); + } + Py_DECREF(fast); + return arr; + } else if (PyDict_Check(v)) { + struct JsonnetJsonValue *obj; + PyObject *key, *val; + Py_ssize_t pos = 0; + obj = jsonnet_json_make_object(vm); + while (PyDict_Next(v, &pos, &key, &val)) { + struct JsonnetJsonValue *json_val; + const char *key_ = PyString_AsString(key); + if (key_ == NULL) { + *err_msg = "Non-string key in dict returned from Python Jsonnet native extension."; + jsonnet_json_destroy(vm, obj); + return NULL; + } + json_val = python_to_jsonnet_json(vm, val, err_msg); + if (json_val == NULL) { + jsonnet_json_destroy(vm, obj); + return NULL; + } + jsonnet_json_object_append(vm, obj, key_, json_val); + } + return obj; + } else { + *err_msg = "Unrecognized type return from Python Jsonnet native extension."; + return NULL; + } +} + /* This function is bound for every native callback, but with a different * context. */ @@ -62,14 +128,15 @@ static struct JsonnetJsonValue *cpython_native_callback( int param_null = jsonnet_json_extract_null(ctx->vm, argv[i]); int param_bool = jsonnet_json_extract_bool(ctx->vm, argv[i]); int param_num = jsonnet_json_extract_number(ctx->vm, argv[i], &d); + PyObject *pyobj; if (param_str != NULL) { - PyTuple_SetItem(arglist, i, PyString_FromString(param_str)); + pyobj = PyString_FromString(param_str); } else if (param_null) { - PyTuple_SetItem(arglist, i, Py_None); + pyobj = Py_None; } else if (param_bool != 2) { - PyTuple_SetItem(arglist, i, PyBool_FromLong(param_bool)); + pyobj = PyBool_FromLong(param_bool); } else if (param_num) { - PyTuple_SetItem(arglist, i, PyFloat_FromDouble(d)); + pyobj = PyFloat_FromDouble(d); } else { // TODO(dcunnin): Support arrays (to tuples). // TODO(dcunnin): Support objects (to dicts). @@ -77,6 +144,7 @@ static struct JsonnetJsonValue *cpython_native_callback( *succ = 0; return jsonnet_json_make_string(ctx->vm, "Non-primitive param."); } + PyTuple_SetItem(arglist, i, pyobj); } // Call python function. @@ -91,23 +159,14 @@ static struct JsonnetJsonValue *cpython_native_callback( return r; } - // TODO(dcunnin): Support arrays (from tuples). - // TODO(dcunnin): Support objects (from dicts). - struct JsonnetJsonValue *r; + const char *err_msg; + struct JsonnetJsonValue *r = python_to_jsonnet_json(ctx->vm, result, &err_msg); + if (r != NULL) { *succ = 1; - if (PyString_Check(result)) { - r = jsonnet_json_make_string(ctx->vm, PyString_AsString(result)); - } else if (PyFloat_Check(result)) { - r = jsonnet_json_make_number(ctx->vm, PyFloat_AsDouble(result)); - } else if (PyBool_Check(result)) { - r = jsonnet_json_make_bool(ctx->vm, PyObject_IsTrue(result)); - } else if (result == Py_None) { - r = jsonnet_json_make_null(ctx->vm); } else { - r = jsonnet_json_make_string(ctx->vm, "Python function did not return primitive"); *succ = 0; + r = jsonnet_json_make_string(ctx->vm, err_msg); } - return r; } From 4ef72cd8d1a34644180e0c226ce40f05eb6832a9 Mon Sep 17 00:00:00 2001 From: Dave Cunningham Date: Thu, 2 Jun 2016 20:47:52 -0400 Subject: [PATCH 3/3] Fix build on Mac --- core/vm.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/vm.cpp b/core/vm.cpp index 255674a73..b9d86ea52 100644 --- a/core/vm.cpp +++ b/core/vm.cpp @@ -2037,8 +2037,8 @@ class Interpreter { JsonnetJsonValue::STRING, encode_utf8(static_cast(arg.v.h)->value), 0, - {}, - {}, + std::vector>{}, + std::map>{}, }); break; @@ -2047,8 +2047,8 @@ class Interpreter { JsonnetJsonValue::BOOL, "", arg.v.b ? 1.0 : 0.0, - {}, - {}, + std::vector>{}, + std::map>{}, }); break; @@ -2057,8 +2057,8 @@ class Interpreter { JsonnetJsonValue::NUMBER, "", arg.v.d, - {}, - {}, + std::vector>{}, + std::map>{}, }); break; @@ -2067,8 +2067,8 @@ class Interpreter { JsonnetJsonValue::NULL_KIND, "", 0, - {}, - {}, + std::vector>{}, + std::map>{}, }); break;