Add importstr, closes #14

Author: sparkprime

ast.h

@@ -35,6 +35,7 @@ enum ASTType {
     AST_ERROR,
     AST_FUNCTION,
     AST_IMPORT,
+    AST_IMPORTSTR,
     AST_INDEX,
     AST_LOCAL,
     AST_LITERAL_BOOLEAN,
@@ -204,14 +205,22 @@ struct Function : public AST {
     { }
 };
 
-/** Represents import e. */
+/** Represents import "file". */
 struct Import : public AST {
     std::string file;
     Import(const LocationRange &lr, const std::string &file)
       : AST(lr, AST_IMPORT), file(file)
     { }
 };
 
+/** Represents importstr "file". */
+struct Importstr : public AST {
+    std::string file;
+    Importstr(const LocationRange &lr, const std::string &file)
+      : AST(lr, AST_IMPORTSTR), file(file)
+    { }
+};
+
 /** Represents both e[e] and the syntax sugar e.f. */
 struct Index : public AST {
     AST *target;

lexer.cpp

@@ -473,6 +473,8 @@ std::list<Token> jsonnet_lex(const std::string &filename, const char *input)
                     kind = Token::IF;
                 } else if (id == "import") {
                     kind = Token::IMPORT;
+                } else if (id == "importstr") {
+                    kind = Token::IMPORTSTR;
                 } else if (id == "in") {
                     kind = Token::IN;
                 } else if (id == "local") {

lexer.h

@@ -54,6 +54,7 @@ struct Token {
         FUNCTION,
         IF,
         IMPORT,
+        IMPORTSTR,
         IN,
         LOCAL,
         NULL_LIT,
@@ -104,6 +105,7 @@ struct Token {
             case FUNCTION: return "function";
             case IF: return "if";
             case IMPORT: return "import";
+            case IMPORTSTR: return "importstr";
             case IN: return "in";
             case LOCAL: return "local";
             case NULL_LIT: return "null";

parser.cpp

@@ -536,6 +536,11 @@ namespace {
                     return alloc.make<Import>(span(tok, file), file.data);
                 }
 
+                case Token::IMPORTSTR: {
+                    Token file = popExpect(Token::STRING);
+                    return alloc.make<Importstr>(span(tok, file), file.data);
+                }
+
 
                 // Variables
                 case Token::DOLLAR:
@@ -994,6 +999,9 @@ static std::string unparse(const AST *ast_)
     } else if (auto *ast = dynamic_cast<const Import*>(ast_)) {
         ss << "import " << jsonnet_unparse_escape(ast->file);
 
+    } else if (auto *ast = dynamic_cast<const Importstr*>(ast_)) {
+        ss << "importstr " << jsonnet_unparse_escape(ast->file);
+
     } else if (auto *ast = dynamic_cast<const Index*>(ast_)) {
         ss << unparse(ast->target) << "["
            << unparse(ast->index) << "]";

static_analysis.cpp

@@ -76,6 +76,9 @@ static IdSet static_analysis(AST *ast_, bool in_object, const IdSet &vars)
     } else if (dynamic_cast<const Import*>(ast_)) {
         // Nothing to do.
 
+    } else if (dynamic_cast<const Importstr*>(ast_)) {
+        // Nothing to do.
+
     } else if (auto *ast = dynamic_cast<const Index*>(ast_)) {
         append(r, static_analysis(ast->target, in_object, vars));
         append(r, static_analysis(ast->index, in_object, vars));

test_suite/import.jsonnet

@@ -20,5 +20,6 @@ std.assertEqual((import "lib/A_20_func.jsonnet")(), 20) &&
 // Each import has its own environment, can't be overidden.
 std.assertEqual(local A = 7; local lib = import "lib/A_20.jsonnet"; lib, 20) &&
 std.assertEqual(local A = 7, lib = import "lib/A_20.jsonnet"; lib, 20) &&
+std.assertEqual(importstr "lib/some_file.txt", "Hello World!\n") &&
 
 true

test_suite/lib/some_file.txt

@@ -0,0 +1 @@
+Hello World!

vim/syntax/jsonnet.vim

@@ -24,6 +24,7 @@ syn match Type "std.length"
 syn match Type "std.join"
 syn match Type "std.objectFields"
 syn match Type "std.filterMap"
+syn match Type "std.makeArray"
 
 syn match Type "\$"
 
@@ -34,7 +35,7 @@ syn match Comment "//.*$"
 
 syn match Keyword "\<[a-zA-Z_][a-z0-9A-Z_]*\w*:"
 
-syntax keyword Special import
+syntax keyword Special import importstr
 syntax keyword Type local function self super if then else for in
 syntax keyword Constant true false null
 syntax keyword Error error

vm.cpp

@@ -372,7 +372,7 @@ namespace {
         const Identifier *idArrayElement;
 
         /** Cache for imported Jsonnet files. */
-        std::map<std::string, AST *> cachedImports;
+        std::map<std::string, const std::string *> cachedImports;
 
         RuntimeError makeError(const LocationRange &loc, const std::string &msg)
         {
@@ -581,22 +581,36 @@ namespace {
          */
         AST *import(const LocationRange &loc, const std::string &file)
         {
-            auto *expr = cachedImports[file];
-            if (expr != nullptr) return expr;
+            const std::string *input = importString(loc, file);
+            auto *expr = jsonnet_parse(alloc, file, input->c_str());
+            jsonnet_static_analysis(expr);
+            return expr;
+        }
 
-            std::string input;
+        /** Import a file as a string.
+         *
+         * If the file has already been imported, then use that version.  This maintains
+         * referential transparency in the case of writes to disk during execution.
+         *
+         * \param loc Location of the import statement.
+         * \param file Path to the filename.
+         */
+        const std::string *importString(const LocationRange &loc, const std::string &file)
+        {
+            const std::string *str = cachedImports[file];
+            if (str != nullptr) return str;
+
+            std::string *input = new std::string();
             std::ifstream f;
             f.open(file.c_str());
             if (!f.good()) {
                 std::string msg = "Couldn't open import \"" + file + "\": ";
                 msg += std::strerror(errno);
                 throw makeError(loc, msg);
             }
-            input.assign(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());
-            expr = jsonnet_parse(alloc, file, input.c_str());
-            jsonnet_static_analysis(expr);
-            cachedImports[file] = expr;
-            return expr;
+            input->assign(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());
+            cachedImports[file] = input;
+            return input;
         }
 
         /** Capture the required variables from the environment. */
@@ -645,6 +659,9 @@ namespace {
         /** Clean up the heap, stack, stash, and builtin function ASTs. */
         ~Interpreter()
         {
+            for (const auto &pair : cachedImports) {
+                delete pair.second;
+            }
         }
 
         const Value &getScratchRegister(void)
@@ -913,6 +930,12 @@ namespace {
                     goto recurse;
                 } break;
 
+                case AST_IMPORTSTR: {
+                    const auto &ast = *static_cast<const Importstr*>(ast_);
+                    const std::string *str = importString(ast.location, ast.file);
+                    scratch = makeString(*str);
+                } break;
+
                 case AST_INDEX: {
                     const auto &ast = *static_cast<const Index*>(ast_);
                     stack.newFrame(FRAME_INDEX_TARGET, ast_);