Merge pull request #204 from sparkprime/extensions

Return arbitrary JSON from native extensions

core/json.h

@@ -17,22 +17,26 @@ limitations under the License.
 #ifndef JSONNET_JSON_H
 #define JSONNET_JSON_H
 
+#include <vector>
+#include <memory>
 #include <string>
 
 #include <libjsonnet.h>
 
 struct JsonnetJsonValue {
     enum Kind {
-        ARRAY,  // Not implemented yet.
+        ARRAY,
         BOOL,
         NULL_KIND,
         NUMBER,
-        OBJECT,  // Not implemented yet.
+        OBJECT,
         STRING,
     };
     Kind kind;
     std::string string;
-    double number;  // Also used for bool (0 and 1)
+    double number;  // Also used for bool (0.0 and 1.0)
+    std::vector<std::unique_ptr<JsonnetJsonValue>> elements;
+    std::map<std::string, std::unique_ptr<JsonnetJsonValue>> fields;
 };
 
 #endif

core/libjsonnet.cpp

@@ -116,6 +116,43 @@ JsonnetJsonValue *jsonnet_json_make_null(struct JsonnetVm *vm)
     return r;
 }
 
+JsonnetJsonValue *jsonnet_json_make_array(JsonnetVm *vm)
+{
+    (void) vm;
+    JsonnetJsonValue *r = new JsonnetJsonValue();
+    r->kind = JsonnetJsonValue::ARRAY;
+    return r;
+}
+
+void jsonnet_json_array_append(JsonnetVm *vm, JsonnetJsonValue *arr, JsonnetJsonValue *v)
+{
+    (void) vm;
+    assert(arr->kind == JsonnetJsonValue::ARRAY);
+    arr->elements.emplace_back(v);
+}
+
+JsonnetJsonValue *jsonnet_json_make_object(JsonnetVm *vm)
+{
+    (void) vm;
+    JsonnetJsonValue *r = new JsonnetJsonValue();
+    r->kind = JsonnetJsonValue::OBJECT;
+    return r;
+}
+
+void jsonnet_json_object_append(JsonnetVm *vm, JsonnetJsonValue *obj,
+                                const char *f, JsonnetJsonValue *v)
+{
+    (void) vm;
+    assert(obj->kind == JsonnetJsonValue::OBJECT);
+    obj->fields[std::string(f)] = std::unique_ptr<JsonnetJsonValue>(v);
+}
+
+void jsonnet_json_destroy(JsonnetVm *vm, JsonnetJsonValue *v)
+{
+    (void) vm;
+    delete v;
+}
+
 struct JsonnetVm {
     double gcGrowthTrigger;
     unsigned maxStack;

core/libjsonnet_test_snippet.c

@@ -21,7 +21,9 @@ limitations under the License.
 
 #include <libjsonnet.h>
 
-struct JsonnetJsonValue *native_concat(void *ctx, const struct JsonnetJsonValue * const *argv, int *succ)
+typedef struct JsonnetJsonValue JJV;
+
+static JJV *native_concat(void *ctx, const JJV * const *argv, int *succ)
 {
     struct JsonnetVm *vm = (struct JsonnetVm *)ctx;
     const char *a = jsonnet_json_extract_string(vm, argv[0]);
@@ -32,13 +34,13 @@ struct JsonnetJsonValue *native_concat(void *ctx, const struct JsonnetJsonValue
     }
     char *str = malloc(strlen(a) + strlen(b) + 1);
     sprintf(str, "%s%s", a, b);
-    struct JsonnetJsonValue *r = jsonnet_json_make_string(vm, str);
+    JJV *r = jsonnet_json_make_string(vm, str);
     free(str);
     *succ = 1;
     return r;
 }
 
-struct JsonnetJsonValue *native_square(void *ctx, const struct JsonnetJsonValue * const *argv, int *succ)
+static JJV *native_square(void *ctx, const JJV * const *argv, int *succ)
 {
     struct JsonnetVm *vm = (struct JsonnetVm *)ctx;
     double a;
@@ -50,11 +52,44 @@ struct JsonnetJsonValue *native_square(void *ctx, const struct JsonnetJsonValue
     return jsonnet_json_make_number(vm, a * a);
 }
 
+static JJV *native_build(void *ctx, const JJV * const *argv, int *succ)
+{
+    struct JsonnetVm *vm = (struct JsonnetVm *)ctx;
+    (void) argv;
+    JJV *obj_top = jsonnet_json_make_object(vm);
+    JJV *arr_top = jsonnet_json_make_array(vm);
+    JJV *arr1 = jsonnet_json_make_array(vm);
+    jsonnet_json_array_append(vm, arr1, jsonnet_json_make_string(vm, "Test 1.1"));
+    jsonnet_json_array_append(vm, arr1, jsonnet_json_make_string(vm, "Test 1.2"));
+    jsonnet_json_array_append(vm, arr1, jsonnet_json_make_string(vm, "Test 1.3"));
+    jsonnet_json_array_append(vm, arr1, jsonnet_json_make_bool(vm, 1));
+    jsonnet_json_array_append(vm, arr1, jsonnet_json_make_number(vm, 42));
+    jsonnet_json_array_append(vm, arr1, jsonnet_json_make_null(vm));
+    jsonnet_json_array_append(vm, arr1, jsonnet_json_make_object(vm));
+    jsonnet_json_array_append(vm, arr_top, arr1);
+    JJV *arr2 = jsonnet_json_make_array(vm);
+    jsonnet_json_array_append(vm, arr2, jsonnet_json_make_string(vm, "Test 2.1"));
+    jsonnet_json_array_append(vm, arr2, jsonnet_json_make_string(vm, "Test 2.2"));
+    jsonnet_json_array_append(vm, arr2, jsonnet_json_make_string(vm, "Test 2.3"));
+    jsonnet_json_array_append(vm, arr2, jsonnet_json_make_bool(vm, 0));
+    jsonnet_json_array_append(vm, arr2, jsonnet_json_make_number(vm, -42));
+    jsonnet_json_array_append(vm, arr2, jsonnet_json_make_null(vm));
+    JJV *little_obj = jsonnet_json_make_object(vm);
+    jsonnet_json_object_append(vm, little_obj, "f", jsonnet_json_make_string(vm, "foo"));
+    jsonnet_json_object_append(vm, little_obj, "g", jsonnet_json_make_string(vm, "bar"));
+    jsonnet_json_array_append(vm, arr2, little_obj);
+    jsonnet_json_array_append(vm, arr_top, arr2);
+    jsonnet_json_object_append(vm, obj_top, "field", arr_top);
+    *succ = 1;
+    return obj_top;
+}
+
 int main(int argc, const char **argv)
 {
     int error;
     char *output;
     struct JsonnetVm *vm;
+    const char *params0[] = {NULL};
     const char *params1[] = {"a", NULL};
     const char *params2[] = {"a", "b", NULL};
     if (argc != 2) {
@@ -64,6 +99,7 @@ int main(int argc, const char **argv)
     vm = jsonnet_make();
     jsonnet_native_callback(vm, "concat", native_concat, vm, params2);
     jsonnet_native_callback(vm, "square", native_square, vm, params1);
+    jsonnet_native_callback(vm, "build", native_build, vm, params0);
     output = jsonnet_evaluate_snippet(vm, "snippet", argv[1], &error);
     if (error) {
         fprintf(stderr, "%s", output);

core/state.h

@@ -209,8 +209,11 @@ struct HeapComprehensionObject : public HeapLeafObject {
      * For each field, holds the value that should be bound to id.  This is the corresponding
      * array element from the original array used to define this object.  This should not really
      * be a thunk, but it makes the implementation easier.
+     *
+     * It is convenient to make this non-const to allow building up the values one by one, so that
+     * the garbage collector can see them at each intermediate point.
      */
-    const std::map<const Identifier*, HeapThunk*> compValues;
+    std::map<const Identifier*, HeapThunk*> compValues;
 
     HeapComprehensionObject(const BindingFrame &up_values, const AST *value,
                             const Identifier *id,

core/vm.cpp

@@ -451,6 +451,12 @@ class Interpreter {
     /** Used to "name" thunks created to execute invariants. */
     const Identifier *idInvariant;
 
+    /** Used to "name" thunks created to convert JSON to Jsonnet objects. */
+    const Identifier *idJsonObjVar;
+
+    /** Used to refer to idJsonObjVar. */
+    const AST *jsonObjVar;
+
     struct ImportCacheValue {
         std::string foundHere;
         std::string content;
@@ -779,6 +785,8 @@ class Interpreter {
         alloc(alloc),
         idArrayElement(alloc->makeIdentifier(U"array_element")),
         idInvariant(alloc->makeIdentifier(U"object_assert")),
+        idJsonObjVar(alloc->makeIdentifier(U"_")),
+        jsonObjVar(alloc->make<Var>(LocationRange(), Fodder{}, idJsonObjVar)),
         externalVars(ext_vars),
         nativeCallbacks(native_callbacks),
         importCallback(import_callback),
@@ -1262,6 +1270,52 @@ class Interpreter {
         return nullptr;
     } 
 
+    void jsonToHeap(const std::unique_ptr<JsonnetJsonValue> &v, Value &attach)
+    {
+        // In order to not anger the garbage collector, assign to attach immediately after
+        // making the heap object.
+        switch (v->kind) {
+            case JsonnetJsonValue::STRING:
+            attach = makeString(decode_utf8(v->string));
+            break;
+
+            case JsonnetJsonValue::BOOL:
+            attach = makeBoolean(v->number != 0.0);
+            break;
+
+            case JsonnetJsonValue::NUMBER:
+            attach = makeDouble(v->number);
+            break;
+
+            case JsonnetJsonValue::NULL_KIND:
+            attach = makeNull();
+            break;
+
+            case JsonnetJsonValue::ARRAY: {
+                attach = makeArray(std::vector<HeapThunk*>{});
+                auto *arr = static_cast<HeapArray*>(attach.v.h);
+                for (size_t i = 0; i < v->elements.size() ; ++i) {
+                    arr->elements.push_back(
+                        makeHeap<HeapThunk>(idArrayElement, nullptr, 0, nullptr));
+                    arr->elements[i]->filled = true;
+                    jsonToHeap(v->elements[i], arr->elements[i]->content);
+                }
+            } break;
+
+            case JsonnetJsonValue::OBJECT: {
+                attach = makeObject<HeapComprehensionObject>(
+                    BindingFrame{}, jsonObjVar, idJsonObjVar, BindingFrame{});
+                auto *obj = static_cast<HeapComprehensionObject*>(attach.v.h);
+                for (const auto &pair : v->fields) {
+                    auto *thunk = makeHeap<HeapThunk>(idJsonObjVar, nullptr, 0, nullptr);
+                    obj->compValues[alloc->makeIdentifier(decode_utf8(pair.first))] = thunk;
+                    thunk->filled = true;
+                    jsonToHeap(pair.second, thunk->content);
+                }
+            } break;
+        }
+    }
+
 
     String toString(const LocationRange &loc)
     {
@@ -1983,14 +2037,18 @@ class Interpreter {
                                     JsonnetJsonValue::STRING,
                                     encode_utf8(static_cast<HeapString*>(arg.v.h)->value),
                                     0,
+                                    std::vector<std::unique_ptr<JsonnetJsonValue>>{},
+                                    std::map<std::string, std::unique_ptr<JsonnetJsonValue>>{},
                                 });
                                 break;
 
                                 case Value::BOOLEAN:
                                 args2.push_back(JsonnetJsonValue{
                                     JsonnetJsonValue::BOOL,
                                     "",
-                                    arg.v.b ? 0.0 : 1.0,
+                                    arg.v.b ? 1.0 : 0.0,
+                                    std::vector<std::unique_ptr<JsonnetJsonValue>>{},
+                                    std::map<std::string, std::unique_ptr<JsonnetJsonValue>>{},
                                 });
                                 break;
 
@@ -1999,6 +2057,8 @@ class Interpreter {
                                     JsonnetJsonValue::NUMBER,
                                     "",
                                     arg.v.d,
+                                    std::vector<std::unique_ptr<JsonnetJsonValue>>{},
+                                    std::map<std::string, std::unique_ptr<JsonnetJsonValue>>{},
                                 });
                                 break;
 
@@ -2007,6 +2067,8 @@ class Interpreter {
                                     JsonnetJsonValue::NULL_KIND,
                                     "",
                                     0,
+                                    std::vector<std::unique_ptr<JsonnetJsonValue>>{},
+                                    std::map<std::string, std::unique_ptr<JsonnetJsonValue>>{},
                                 });
                                 break;
 
@@ -2028,31 +2090,9 @@ class Interpreter {
 
                         int succ;
                         std::unique_ptr<JsonnetJsonValue> r(cb.cb(cb.ctx, &args3[0], &succ));
-                        if (succ) {
-                            // TODO(dcunnin): Support arrays.
-                            // TODO(dcunnin): Support objects.
-                            switch (r->kind) {
-                                case JsonnetJsonValue::STRING:
-                                scratch = makeString(decode_utf8(r->string));
-                                break;
-
-                                case JsonnetJsonValue::BOOL:
-                                scratch = makeBoolean(r->number != 0.0);
-                                break;
-
-                                case JsonnetJsonValue::NUMBER:
-                                scratch = makeDouble(r->number);
-                                break;
-
-                                case JsonnetJsonValue::NULL_KIND:
-                                scratch = makeNull();
-                                break;
-
-                                default:
-                                throw makeError(ast.location,
-                                                "Native extensions can only return primitives.");
-                            }
 
+                        if (succ) {
+                            jsonToHeap(r, scratch);
                         } else {
                             if (r->kind != JsonnetJsonValue::STRING) {
                                 throw makeError(

include/libjsonnet.h

@@ -104,6 +104,39 @@ struct JsonnetJsonValue *jsonnet_json_make_bool(struct JsonnetVm *vm, int v);
  */
 struct JsonnetJsonValue *jsonnet_json_make_null(struct JsonnetVm *vm);
 
+/** Make a JsonnetJsonValue representing an array.
+ *
+ * Assign elements with jsonnet_json_array_append.
+ */
+struct JsonnetJsonValue *jsonnet_json_make_array(struct JsonnetVm *vm);
+
+/** Add v to the end of the array.
+ */
+void jsonnet_json_array_append(struct JsonnetVm *vm,
+                               struct JsonnetJsonValue *arr,
+                               struct JsonnetJsonValue *v);
+
+/** Make a JsonnetJsonValue representing an object with the given number of fields.
+ *
+ * Every index of the array must have a unique value assigned with jsonnet_json_array_element.
+ */
+struct JsonnetJsonValue *jsonnet_json_make_object(struct JsonnetVm *vm);
+
+/** Add the field f to the object, bound to v.
+ *
+ * This replaces any previous binding of the field.
+ */
+void jsonnet_json_object_append(struct JsonnetVm *vm,
+                                struct JsonnetJsonValue *obj,
+                                const char *f,
+                                struct JsonnetJsonValue *v);
+
+/** Clean up a JSON subtree.
+ *
+ * This is useful if you want to abort with an error mid-way through building a complex value.
+ */
+void jsonnet_json_destroy(struct JsonnetVm *vm, struct JsonnetJsonValue *v);
+
 /** Callback to provide native extensions to Jsonnet.
  *
  * The returned JsonnetJsonValue* should be allocated with jsonnet_realloc.  It will be cleaned up