From 0299d84a5aa4c35c346125fa6ac31a14d6a923ff Mon Sep 17 00:00:00 2001 From: Dave Cunningham Date: Wed, 2 Sep 2015 02:05:02 -0400 Subject: [PATCH] Add a way to supply structured data via external variables. --- doc/commandline.html | 4 ++++ doc/commandline.html.jinja | 4 ++++ jsonnet.cpp | 22 ++++++++++++++++++++++ libjsonnet.cpp | 17 +++++++++++------ libjsonnet.h | 6 ++++++ test_suite/run_tests.sh | 2 +- test_suite/stdlib.jsonnet | 8 ++++++++ vm.cpp | 24 +++++++++++++++++++----- vm.h | 17 +++++++++++++++-- 9 files changed, 90 insertions(+), 14 deletions(-) diff --git a/doc/commandline.html b/doc/commandline.html index c8eaaaebc..c642b99ca 100644 --- a/doc/commandline.html +++ b/doc/commandline.html @@ -216,9 +216,13 @@

Usage

-J / --jpath <dir> Specify an additional library search dir -V / --var <var>=<val> Specify an 'external' var to the given value -E / --env <var> Bring in an environment var as an 'external' var + --code-var <var>=<val> As --var but value is Jsonnet code + --code-env <var> As --env but env var contains Jsonnet code -m / --multi Write multiple files, list files on stdout + -S / --string Expect a string, manifest as plain text -s / --max-stack <n> Number of allowed stack frames -t / --max-trace <n> Max length of stack trace before cropping + --gc-min-objects <n> Do not run garbage collector until this many --gc-growth-trigger <n> Run garbage collector after this amount of object growth --debug-ast Unparse the parsed AST without executing it diff --git a/doc/commandline.html.jinja b/doc/commandline.html.jinja index de34064f6..2a6c7ac82 100644 --- a/doc/commandline.html.jinja +++ b/doc/commandline.html.jinja @@ -27,9 +27,13 @@ and <option> can be: -J / --jpath <dir> Specify an additional library search dir -V / --var <var>=<val> Specify an 'external' var to the given value -E / --env <var> Bring in an environment var as an 'external' var + --code-var <var>=<val> As --var but value is Jsonnet code + --code-env <var> As --env but env var contains Jsonnet code -m / --multi Write multiple files, list files on stdout + -S / --string Expect a string, manifest as plain text -s / --max-stack <n> Number of allowed stack frames -t / --max-trace <n> Max length of stack trace before cropping + --gc-min-objects <n> Do not run garbage collector until this many --gc-growth-trigger <n> Run garbage collector after this amount of object growth --debug-ast Unparse the parsed AST without executing it diff --git a/jsonnet.cpp b/jsonnet.cpp index 4224e060a..c8cc34a26 100644 --- a/jsonnet.cpp +++ b/jsonnet.cpp @@ -174,6 +174,8 @@ void usage(std::ostream &o) o << " -J / --jpath Specify an additional library search dir\n"; o << " -V / --var = Specify an 'external' var to the given value\n"; o << " -E / --env Bring in an environment var as an 'external' var\n"; + o << " --code-var = As --var but value is Jsonnet code\n"; + o << " --code-env As --env but env var contains Jsonnet code\n"; o << " -m / --multi Write multiple files, list files on stdout\n"; o << " -S / --string Expect a string, manifest as plain text\n"; o << " -s / --max-stack Number of allowed stack frames\n"; @@ -262,6 +264,26 @@ int main(int argc, const char **argv) const std::string var = var_val.substr(0, eq_pos); const std::string val = var_val.substr(eq_pos + 1, std::string::npos); jsonnet_ext_var(vm, var.c_str(), val.c_str()); + } else if (arg == "--code-env") { + const std::string var = next_arg(i, args); + const char *val = ::getenv(var.c_str()); + if (val == nullptr) { + std::cerr << "ERROR: Environment variable " << var + << " was undefined." << std::endl; + return EXIT_FAILURE; + } + jsonnet_ext_code(vm, var.c_str(), val); + } else if (arg == "--code-var") { + const std::string var_val = next_arg(i, args); + size_t eq_pos = var_val.find_first_of('=', 0); + if (eq_pos == std::string::npos) { + std::cerr << "ERROR: argument not in form = \"" + << var_val << "\"." << std::endl; + return EXIT_FAILURE; + } + const std::string var = var_val.substr(0, eq_pos); + const std::string val = var_val.substr(eq_pos + 1, std::string::npos); + jsonnet_ext_code(vm, var.c_str(), val.c_str()); } else if (arg == "--gc-min-objects") { long l = strtol_check(next_arg(i, args)); if (l < 0) { diff --git a/libjsonnet.cpp b/libjsonnet.cpp index faec832d7..d6458958a 100644 --- a/libjsonnet.cpp +++ b/libjsonnet.cpp @@ -86,20 +86,20 @@ static char *default_import_callback(void *ctx, const char *base, const char *fi } } - struct JsonnetVm { double gcGrowthTrigger; unsigned maxStack; unsigned gcMinObjects; bool debugAst; unsigned maxTrace; - std::map extVars; + std::map ext; JsonnetImportCallback *importCallback; void *importCallbackContext; bool stringOutput; JsonnetVm(void) : gcGrowthTrigger(2.0), maxStack(500), gcMinObjects(1000), debugAst(false), maxTrace(20), - importCallback(default_import_callback), importCallbackContext(this), stringOutput(false) + importCallback(default_import_callback), importCallbackContext(this), + stringOutput(false) { } }; @@ -161,7 +161,12 @@ void jsonnet_import_callback(struct JsonnetVm *vm, JsonnetImportCallback *cb, vo void jsonnet_ext_var(JsonnetVm *vm, const char *key, const char *val) { - vm->extVars[key] = val; + vm->ext[key] = VmExt(val, false); +} + +void jsonnet_ext_code(JsonnetVm *vm, const char *key, const char *val) +{ + vm->ext[key] = VmExt(val, true); } void jsonnet_debug_ast(JsonnetVm *vm, int v) @@ -187,12 +192,12 @@ static char *jsonnet_evaluate_snippet_aux(JsonnetVm *vm, const char *filename, } else { jsonnet_static_analysis(expr); if (multi) { - files = jsonnet_vm_execute_multi(&alloc, expr, vm->extVars, vm->maxStack, + files = jsonnet_vm_execute_multi(&alloc, expr, vm->ext, vm->maxStack, vm->gcMinObjects, vm->gcGrowthTrigger, vm->importCallback, vm->importCallbackContext, vm->stringOutput); } else { - json_str = jsonnet_vm_execute(&alloc, expr, vm->extVars, vm->maxStack, + json_str = jsonnet_vm_execute(&alloc, expr, vm->ext, vm->maxStack, vm->gcMinObjects, vm->gcGrowthTrigger, vm->importCallback, vm->importCallbackContext, vm->stringOutput); diff --git a/libjsonnet.h b/libjsonnet.h index f47e5b42e..f6e0f2de4 100644 --- a/libjsonnet.h +++ b/libjsonnet.h @@ -85,6 +85,12 @@ void jsonnet_import_callback(struct JsonnetVm *vm, JsonnetImportCallback *cb, vo */ void jsonnet_ext_var(struct JsonnetVm *vm, const char *key, const char *val); +/** Bind a Jsonnet external code var to the given value. + * + * Argument values are copied so memory should be managed by caller. + */ +void jsonnet_ext_code(struct JsonnetVm *vm, const char *key, const char *val); + /** If set to 1, will emit the Jsonnet input after parsing / desugaring. */ void jsonnet_debug_ast(struct JsonnetVm *vm, int v); diff --git a/test_suite/run_tests.sh b/test_suite/run_tests.sh index c4ef0aed2..a1ce7242f 100755 --- a/test_suite/run_tests.sh +++ b/test_suite/run_tests.sh @@ -26,7 +26,7 @@ for TEST in *.jsonnet ; do if [ -r "$TEST.golden" ] ; then GOLDEN=$(cat "$TEST.golden") fi - OUTPUT="$(../jsonnet --gc-min-objects 1 --gc-growth-trigger 1 "$TEST" 2>&1 )" + OUTPUT="$(../jsonnet --gc-min-objects 1 --gc-growth-trigger 1 --var var1=test --code-var var2='{x:1, y: 2}' "$TEST" 2>&1 )" EXIT_CODE=$? if [ $EXIT_CODE -ne $EXPECTED_EXIT_CODE ] ; then FAILED=$((FAILED + 1)) diff --git a/test_suite/stdlib.jsonnet b/test_suite/stdlib.jsonnet index a6dc8bfb5..f6f2a1a7f 100644 --- a/test_suite/stdlib.jsonnet +++ b/test_suite/stdlib.jsonnet @@ -304,4 +304,12 @@ std.assertEqual(std.setMember("a", ["b", "c"]), false) && std.assertEqual(std.thisFile, "stdlib.jsonnet") && std.assertEqual(import "this_file/a.jsonnet", "this_file/a.jsonnet") && std.assertEqual(import "this_file/b.jsonnet", "this_file/a.jsonnet") && + + +std.assertEqual(std.extVar("var1"), "test") && + +std.assertEqual(std.toString(std.extVar("var2")), "{\"x\": 1, \"y\": 2}") && +std.assertEqual(std.extVar("var2"), {x: 1, y: 2}) && +std.assertEqual(std.extVar("var2") { x+: 2}.x, 3) && + true diff --git a/vm.cpp b/vm.cpp index 7f807cdee..1469f3e85 100644 --- a/vm.cpp +++ b/vm.cpp @@ -43,6 +43,7 @@ namespace { FRAME_APPLY_TARGET, FRAME_BINARY_LEFT, FRAME_BINARY_RIGHT, + FRAME_BUILTIN_EXT_VAR, FRAME_BUILTIN_FILTER, FRAME_BUILTIN_FORCE_THUNKS, FRAME_CALL, @@ -379,6 +380,9 @@ namespace { } }; + /** Typedef to save some typing. */ + typedef std::map ExtMap; + /** Typedef to save some typing. */ typedef std::map StrMap; @@ -421,7 +425,7 @@ namespace { const ImportCacheValue *> cachedImports; /** External variables for std.extVar. */ - StrMap externalVars; + ExtMap externalVars; /** The callback used for loading imported files. */ JsonnetImportCallback *importCallback; @@ -735,7 +739,7 @@ namespace { * * \param loc The location range of the file to be executed. */ - Interpreter(Allocator *alloc, const StrMap &ext_vars, + Interpreter(Allocator *alloc, const ExtMap &ext_vars, unsigned max_stack, double gc_min_objects, double gc_growth_trigger, JsonnetImportCallback *import_callback, void *import_callback_context) : heap(gc_min_objects, gc_growth_trigger), stack(max_stack), alloc(alloc), @@ -1756,7 +1760,17 @@ namespace { if (externalVars.find(var) == externalVars.end()) { throw makeError(ast.location, "Undefined external variable: " + var); } - scratch = makeString(externalVars[var]); + const VmExt &ext = externalVars[var]; + if (ext.isCode) { + std::string filename = ""; + AST *expr = jsonnet_parse(alloc, filename, ext.data.c_str()); + jsonnet_static_analysis(expr); + ast_ = expr; + stack.pop(); + goto recurse; + } else { + scratch = makeString(ext.data); + } } break; default: @@ -2217,7 +2231,7 @@ namespace { } std::string jsonnet_vm_execute(Allocator *alloc, const AST *ast, - const StrMap &ext_vars, + const ExtMap &ext_vars, unsigned max_stack, double gc_min_objects, double gc_growth_trigger, JsonnetImportCallback *import_callback, void *ctx, @@ -2233,7 +2247,7 @@ std::string jsonnet_vm_execute(Allocator *alloc, const AST *ast, } } -StrMap jsonnet_vm_execute_multi(Allocator *alloc, const AST *ast, const StrMap &ext_vars, +StrMap jsonnet_vm_execute_multi(Allocator *alloc, const AST *ast, const ExtMap &ext_vars, unsigned max_stack, double gc_min_objects, double gc_growth_trigger, JsonnetImportCallback *import_callback, void *ctx, bool string_output) diff --git a/vm.h b/vm.h index d7204e9ed..e80e09f17 100644 --- a/vm.h +++ b/vm.h @@ -41,10 +41,22 @@ struct RuntimeError { { } }; +/** Stores external values / code. */ +struct VmExt { + std::string data; + bool isCode; + VmExt() : isCode(false) { } + VmExt(const std::string &data, bool is_code) + : data(data), isCode(is_code) + { } +}; + + /** Execute the program and return the value as a JSON string. * * \param alloc The allocator used to create the ast. * \param ast The program to execute. + * \param ext The external vars / code. * \param max_stack Recursion beyond this level gives an error. * \param gc_min_objects The garbage collector does not run when the heap is this small. * \param gc_growth_trigger Growth since last garbage collection cycle to trigger a new cycle. @@ -55,7 +67,7 @@ struct RuntimeError { * \returns The JSON result in string form. */ std::string jsonnet_vm_execute(Allocator *alloc, const AST *ast, - const std::map &ext_vars, + const std::map &ext, unsigned max_stack, double gc_min_objects, double gc_growth_trigger, JsonnetImportCallback *import_callback, void *import_callback_ctx, @@ -67,6 +79,7 @@ std::string jsonnet_vm_execute(Allocator *alloc, const AST *ast, * * \param alloc The allocator used to create the ast. * \param ast The program to execute. + * \param ext The external vars / code. * \param max_stack Recursion beyond this level gives an error. * \param gc_min_objects The garbage collector does not run when the heap is this small. * \param gc_growth_trigger Growth since last garbage collection cycle to trigger a new cycle. @@ -77,7 +90,7 @@ std::string jsonnet_vm_execute(Allocator *alloc, const AST *ast, * \returns A mapping from filename to the JSON strings for that file. */ std::map jsonnet_vm_execute_multi( - Allocator *alloc, const AST *ast, const std::map &ext_vars, + Allocator *alloc, const AST *ast, const std::map &ext, unsigned max_stack, double gc_min_objects, double gc_growth_trigger, JsonnetImportCallback *import_callback, void *import_callback_ctx, bool string_output);