diff --git a/.gitmodules b/.gitmodules index 3d00fe8f178..0b6d96238ff 100644 --- a/.gitmodules +++ b/.gitmodules @@ -41,3 +41,6 @@ [submodule "external/WindowsToolchain"] path = external/WindowsToolchain url = https://github.com/MarkSchofield/WindowsToolchain/ +[submodule "external/lua"] + path = external/lua + url = https://github.com/lua/lua.git diff --git a/external/lua b/external/lua new file mode 160000 index 00000000000..3fe7be956f2 --- /dev/null +++ b/external/lua @@ -0,0 +1 @@ +Subproject commit 3fe7be956f23385aa1950dc31e2f25127ccfc0ea diff --git a/source/compiler-core/slang-lexer.cpp b/source/compiler-core/slang-lexer.cpp index 84a4df93bb1..048c266ca48 100644 --- a/source/compiler-core/slang-lexer.cpp +++ b/source/compiler-core/slang-lexer.cpp @@ -1810,6 +1810,18 @@ TokenList Lexer::lexAllMarkupTokens() } } +TokenList Lexer::lexAllTokens() +{ + TokenList tokenList; + for (;;) + { + Token token = lexToken(); + tokenList.add(token); + if (token.type == TokenType::EndOfFile) + return tokenList; + } +} + /* static */ UnownedStringSlice Lexer::sourceLocationLexer(const UnownedStringSlice& in) { Lexer lexer; diff --git a/source/compiler-core/slang-lexer.h b/source/compiler-core/slang-lexer.h index a36719ab79e..0152ff6f55a 100644 --- a/source/compiler-core/slang-lexer.h +++ b/source/compiler-core/slang-lexer.h @@ -139,6 +139,9 @@ struct Lexer /// Lex all tokens (up to the end of the stream) that are relevant to things like markup TokenList lexAllMarkupTokens(); + /// Lex all tokens (up to the end of the stream) whether relevant or not. + TokenList lexAllTokens(); + /// Get the diagnostic sink, taking into account flags. Will return null if suppressing /// diagnostics. DiagnosticSink* getDiagnosticSink() diff --git a/source/slang-core-module/CMakeLists.txt b/source/slang-core-module/CMakeLists.txt index 50b45c90c9b..ba70d77b9b3 100644 --- a/source/slang-core-module/CMakeLists.txt +++ b/source/slang-core-module/CMakeLists.txt @@ -60,6 +60,7 @@ set(core_module_source_common_args LINK_WITH_PRIVATE core slang-capability-defs + slang-fiddle-output slang-reflect-headers SPIRV-Headers INCLUDE_DIRECTORIES_PRIVATE diff --git a/source/slang/CMakeLists.txt b/source/slang/CMakeLists.txt index 9c51ed767e9..2adc96939b5 100644 --- a/source/slang/CMakeLists.txt +++ b/source/slang/CMakeLists.txt @@ -1,3 +1,48 @@ +# +# Invoke the "fiddle" tool to generate source +# + +set(SLANG_FIDDLE_INPUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}") +set(SLANG_FIDDLE_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/fiddle") + +file(GLOB SLANG_FIDDLE_INPUT_FILE_NAMES + CONFIGURE_DEPENDS + RELATIVE "${SLANG_FIDDLE_INPUT_DIR}" + "*.h" + "*.cpp") + +list(TRANSFORM SLANG_FIDDLE_INPUT_FILE_NAMES + PREPEND "${SLANG_FIDDLE_INPUT_DIR}/" + OUTPUT_VARIABLE SLANG_FIDDLE_INPUTS) + +list(TRANSFORM SLANG_FIDDLE_INPUT_FILE_NAMES + APPEND ".fiddle" + OUTPUT_VARIABLE SLANG_FIDDLE_OUTPUTS) +list(TRANSFORM SLANG_FIDDLE_OUTPUTS + PREPEND "${SLANG_FIDDLE_OUTPUT_DIR}/") + +add_custom_command( + OUTPUT ${SLANG_FIDDLE_OUTPUTS} + COMMAND ${CMAKE_COMMAND} -E make_directory ${SLANG_FIDDLE_OUTPUT_DIR} + COMMAND slang-fiddle + -i "${SLANG_FIDDLE_INPUT_DIR}/" + -o "${SLANG_FIDDLE_OUTPUT_DIR}/" + ${SLANG_FIDDLE_INPUT_FILE_NAMES} + DEPENDS ${SLANG_FIDDLE_INPUTS} slang-fiddle + VERBATIM +) +add_library( + slang-fiddle-output + INTERFACE + EXCLUDE_FROM_ALL + ${SLANG_FIDDLE_OUTPUTS} +) +set_target_properties(slang-fiddle-output PROPERTIES FOLDER generated) +target_include_directories( + slang-fiddle-output + INTERFACE ${SLANG_FIDDLE_OUTPUT_DIR} +) + # # generate capability files # @@ -233,6 +278,7 @@ set(slang_link_args compiler-core slang-capability-defs slang-capability-lookup + slang-fiddle-output slang-reflect-headers slang-lookup-tables SPIRV-Headers diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index d558f299e0f..1abbd04ffef 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -45,6 +45,7 @@ function(generator dir) endif() endfunction() +generator(slang-fiddle LINK_WITH_PRIVATE compiler-core) generator( slang-cpp-extractor USE_FEWER_WARNINGS @@ -95,6 +96,7 @@ if(SLANG_ENABLE_SLANGD) compiler-core slang slang-reflect-headers + slang-fiddle-output slang-capability-defs Threads::Threads INSTALL diff --git a/tools/slang-fiddle/README.md b/tools/slang-fiddle/README.md new file mode 100644 index 00000000000..19bda6e8dc2 --- /dev/null +++ b/tools/slang-fiddle/README.md @@ -0,0 +1,155 @@ +Fiddle +====== +> "Fiddle writes code, so you don't have to" + +The `slang-fiddle` tool bundles together a few different pieces of functionality related to source-code generation, in a way that is intended to fit the workflow needed for building the Slang compiler. + +Invoking Fiddle +--------------- + +Fiddle gets invoked from the command line with a command line like: + +``` +slang-fiddle -i source/ -o generated/ a.cpp b.h c.h d.cpp ... +``` + +This will run Fiddle on each of the files `a.cpp`, `b.h`, and so forth. + +The `-i` option gives a prefix (here `source/`) that gets prepended on each of the file names to produce the input path for that file, and the `-o` options gives a prefix (here `generated/`) that gets prepended on each of the file names as part of the output path for that file. The output path for each file is *also* suffixed with `.fiddle`. Thus, the above command line will process `source/a.cpp` to generate `generated/a.cpp.fiddle`, and so on. + +In addition to generating an output file corresponding to each input, Fiddle may *also* overwrite the input file, in order to inject a little bit of code (more on that later). + +For the most part, you shouldn't have to think about how Fiddle gets invoked, because the build system should be doing it for you. + +Overview of Steps +----------------- + +Fiddle does its work across a few different steps: + +* First, each of the input files is read in and parsed for two kinds of constructs: + + * C++ declarations that have been marked with the `FIDDLE()` macro are *scraped* to extract (minimal) information about them. + + * Lines that have `FIDDLE TEMPLATE` markers on them are used to identify the *text templates* for which output should be generated. + +* Second, a minimal amount of semantic checking is performed on the C++ declarations that were scraped (basically just building an inheritance hierarchy). This step happens after *all* of the input files have been scraped, so it can detect relationships between types in different files. + +* Third, code is generated into the output file, for each of the two kinds of constructs: + + * For each `FIDDLE()` macro invocation site, a specialized macro definition is generated that provides the output to match that site + + * For each text template, the template is evaluated (using Lua) to generate the desired code. + +* Fourth, each input file that contained any text templates is carefully overwritten so that the template definition site will `#include` the corresponding generated code from the output file. + +The remaining sections here will go into more detail about the two kinds of constructs. + +Scraping with `FIDDLE()` +------------------------ + +During the scraping step, Fiddle will run the Slang lexer on each of the input files (relying on the fact that the Slang lexer can handle a reasonable subset of C++), and then scan through the token stream produced by the lexer looking for invocations of the `FIDDLE()` macro. + +Putting `FIDDLE()` in front of an ordinary C++ `class`, `struct`, or `namespace` declaration tells Fiddle that it should include that C++ construct in the model of the program that it scrapes. For example, given this input: + +``` +FIDDLE() +namespace MyProgram +{ + FIDDLE() + struct A { /* ... */ }; + + struct B { /* ... */ }; +} +``` + +Fiddle will include the `MyProgram` namespace and the `MyProgram::A` type in its model, but will *not* include `B`. + +> Note that because the scraping step does *not* try to understand anything in the C++ code that is not preceded by an invocation of `FIDDLE()`. + +A programmer can place Fiddle-specific modifiers inside the `FIDDLE()` invocation before a type, to apply those modifiers to the model of that type: + +``` +FIDDLE(abstract) +class Thing { /* ... */ }; + +FIDDLE() +class ConcreteThing : public Thing { /* ... */ } +``` + +One important constraint is that any `struct` or `class` type marked with `FIDDLE()` *must* have an invocation of the form `FIDDLE(...)` (that is, `FIDDLE` applied to an actual ellipsis `...`) as the first item after the opening curly brace for its body: + +``` +FIDDLE() +struct A +{ + FIDDLE(...) + + /* rest of declaration */ +}; +``` + +Fiddle will generate macro definitions that each of the different `FIDDLE()` invocations will expand to (using `__LINE__` so that each can expand differently), and the `FIDDLE(...)` within a type is used to inject additional members into that type. + +> The ability to inject additional declarations in a type is currently mostly used to add all the boilerplate that is required by AST node classes. + +Fields within a type can also be marked with `FIDDLE()` so that they will be available in the data model that Fiddle builds while scraping. +Note again that Fiddle will *ignore* any fields not marked with `FIDDLE()`, so be sure to mark all the fields that are relevant to your purpose. + +In order for Fiddle to provide the macros that each `FIDDLE()` invocation expands into, any input file that includes invocations of `FIDDLE()` *must* also `#include` the corresponding generated file. +For example, the input file `a.cpp` should `#include "a.cpp.fiddle"`. +The `#include` of the generated output file should come *after* any other `#include`s, to make sure that any `.h` files that also use `FIDDLE()` don't cause confusion. + +Text Templates +-------------- + +The real meat of what Fiddle can do is around its support for text templates. These allow your C++ code files (or any text files, really) to embed Lua script code that generates additional C++ source. + +The start of a text template is identified by a line that contains the exact text `FIDDLE TEMPLATE`. +Every text template must follow this sequence: + +* A line containing `FIDDLE TEMPLATE` + +* Zero or more lines of template code + +* A line containing `FIDDLE OUTPUT` + +* Zero or more lines of (generated) output code + +* A line containing `FIDDLE END` + +Fiddle doesn't care what else is on the three marker lines it uses, so you can construct your code to conveniently place the markers and the template in comments or other code that the C++ compiler won't see. +An idiomatic approach would be something like: + +``` +// my-class-forward-decls.h +#pragma once + +// Generate a bunch of forward declarations: +// +#if 0 // FIDDLE TEMPLATE: +%for _,T in ipairs(MyNamespace.MyClass.subclasses) do + class $T; +%end +#else // FIDDLE OUTPUT: +#endif +``` + +For the template part of things, you can write lines of more-or-less ordinary C++ code, interspersed with two kinds of script code: + +* Lines in a template where `%` is the first non-whitespace character are assumed to be Lua statements + +* Otherwise, any `$` in a line marks the start of a *splice*. The `$` can be followed by a single identifier, or a Lua expression enclosed in `()` (the nested expression *must* have balanced `()`s). + +Statement lines (using `%`) are executed for their effect, and don't directly produce output, while splices (using `$`) evaluate a Lua expression and then apply `tostring()` to it to yield text to be spliced in. + +Rather than directly writing the generated code for a template back into the input file, Fiddle writes the code for each template out in the generated output file, and then injects a simple `#include` at each text template site that pulls in the corresponding text. +For example, given the input above, the generated output for the template might be: + +``` +#if 0 // FIDDLE TEMPLATE: +... +#else // FIDDLE OUTPUT: +#define FIDDLE_TEMPLATE_OUTPUT_ID 0 +#include "my-class-forward-decls.h.fiddle" +#endif +``` diff --git a/tools/slang-fiddle/slang-fiddle-diagnostic-defs.h b/tools/slang-fiddle/slang-fiddle-diagnostic-defs.h new file mode 100644 index 00000000000..204185b1c5e --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-diagnostic-defs.h @@ -0,0 +1,71 @@ +// + +// The file is meant to be included multiple times, to produce different +// pieces of declaration/definition code related to diagnostic messages +// +// Each diagnostic is declared here with: +// +// DIAGNOSTIC(id, severity, name, messageFormat) +// +// Where `id` is the unique diagnostic ID, `severity` is the default +// severity (from the `Severity` enum), `name` is a name used to refer +// to this diagnostic from code, and `messageFormat` is the default +// (non-localized) message for the diagnostic, with placeholders +// for any arguments. + +#ifndef DIAGNOSTIC +#error Need to #define DIAGNOSTIC(...) before including "slang-cpp-parser/diagnostics-defs.h" +#define DIAGNOSTIC(id, severity, name, messageFormat) /* */ +#endif + +#if 0 +DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'") +DIAGNOSTIC(-1, Note, seeOpen, "see open $0") +DIAGNOSTIC(-1, Note, commandLine, "Command line: $0") +DIAGNOSTIC(-1, Note, previousLocation, "previous location") +#endif + +// Command Line + +DIAGNOSTIC(100001, Error, unknownOption, "unknown option '$0'") + +// Basic I/O + +DIAGNOSTIC(200001, Error, couldNotReadInputFile, "could not read input file '$0'") +DIAGNOSTIC(200002, Error, couldNotOverwriteInputFile, "could not overwrite input file '$0'") +DIAGNOSTIC(200002, Error, couldNotWriteOutputFile, "could not write output file '$0'") + +// Template Parsing + +DIAGNOSTIC( + 300001, + Error, + expectedOutputStartMarker, + "start line for template not followed by a line marking output with '$0'") +DIAGNOSTIC(300002, Error, expectedEndMarker, "expected a template end line ('$0')") + +// Scraper: Parsing + +DIAGNOSTIC(500001, Error, unexpected, "unexpected $0, expected $1") + +DIAGNOSTIC( + 501001, + Error, + expectedFiddleEllipsisInvocation, + "expected 'FIDDLE(...)' at start of body of '$0'") + +DIAGNOSTIC( + 502001, + Error, + expectedIncludeOfOutputHeader, + "expected a '#include' of generated output file '$0' in file containing 'FIDDLE(...)' " + "invocations") + +// Scraper: Semantic Checking + +DIAGNOSTIC(600001, Error, undefinedIdentifier, "undefined identifier '$0'") + + +DIAGNOSTIC(999999, Fatal, internalError, "internal error in 'fiddle' tool") + +#undef DIAGNOSTIC diff --git a/tools/slang-fiddle/slang-fiddle-diagnostics.cpp b/tools/slang-fiddle/slang-fiddle-diagnostics.cpp new file mode 100644 index 00000000000..7b59224c6d1 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-diagnostics.cpp @@ -0,0 +1,14 @@ +// slang-fiddle-diagnostics.cpp +#include "slang-fiddle-diagnostics.h" + +namespace fiddle +{ +namespace Diagnostics +{ +using namespace Slang; + +#define DIAGNOSTIC(id, severity, name, messageFormat) \ + const DiagnosticInfo name = {id, Severity::severity, #name, messageFormat}; +#include "slang-fiddle-diagnostic-defs.h" +} // namespace Diagnostics +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-diagnostics.h b/tools/slang-fiddle/slang-fiddle-diagnostics.h new file mode 100644 index 00000000000..6dac91f9567 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-diagnostics.h @@ -0,0 +1,18 @@ +// slang-fiddle-diagnostics.h +#pragma once + +#include "compiler-core/slang-diagnostic-sink.h" +#include "slang/slang-diagnostics.h" + + +namespace fiddle +{ +using namespace Slang; + +namespace Diagnostics +{ + +#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name; +#include "slang-fiddle-diagnostic-defs.h" +} // namespace Diagnostics +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-lua.cpp b/tools/slang-fiddle/slang-fiddle-lua.cpp new file mode 100644 index 00000000000..f6ba36357ad --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-lua.cpp @@ -0,0 +1,5 @@ +// slang-fiddle-lua.cpp + + +#define MAKE_LIB 1 +#include "../external/lua/onelua.c" diff --git a/tools/slang-fiddle/slang-fiddle-main.cpp b/tools/slang-fiddle/slang-fiddle-main.cpp new file mode 100644 index 00000000000..b8541245477 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-main.cpp @@ -0,0 +1,445 @@ +// slang-fiddle-main.cpp + +#include "core/slang-io.h" +#include "slang-fiddle-diagnostics.h" +#include "slang-fiddle-options.h" +#include "slang-fiddle-scrape.h" +#include "slang-fiddle-template.h" + +#if 0 +#include "compiler-core/slang-doc-extractor.h" +#include "compiler-core/slang-name-convention-util.h" +#include "compiler-core/slang-name.h" +#include "compiler-core/slang-source-loc.h" +#include "core/slang-file-system.h" +#include "core/slang-list.h" +#include "core/slang-secure-crt.h" +#include "core/slang-string-slice-pool.h" +#include "core/slang-string-util.h" +#include "core/slang-string.h" +#include "core/slang-writer.h" +#include "slang-com-helper.h" + +#include +#include +#include +#endif + + +namespace fiddle +{ +using namespace Slang; + +class InputFile : public RefObject +{ +public: + String inputFileName; + + RefPtr scrapedSourceUnit; + RefPtr textTemplateFile; +}; + +struct App +{ +public: + App(SourceManager& sourceManager, DiagnosticSink& sink, RootNamePool& rootNamePool) + : sourceManager(sourceManager), sink(sink), rootNamePool(rootNamePool) + { + } + + RootNamePool& rootNamePool; + SourceManager& sourceManager; + DiagnosticSink& sink; + + Options options; + + List> inputFiles; + RefPtr logicalModule; + + RefPtr parseSourceUnit(SourceView* inputSourceView, String outputFileName) + { + return fiddle::parseSourceUnit( + inputSourceView, + logicalModule, + &rootNamePool, + &sink, + &sourceManager, + outputFileName); + } + + RefPtr parseTextTemplate(SourceView* inputSourceView) + { + return fiddle::parseTextTemplateFile(inputSourceView, &sink); + } + + String getOutputFileName(String inputFileName) { return inputFileName + ".fiddle"; } + + void processInputFile(String const& inputFileName) + { + // The full path to the input and output is determined by the prefixes that + // were specified via command-line arguments. + // + String inputPath = options.inputPathPrefix + inputFileName; + + // We read the fill text of the file into memory as a single string, + // so that we can easily parse it without need for I/O operations + // along the way. + // + String inputText; + if (SLANG_FAILED(File::readAllText(inputPath, inputText))) + { + sink.diagnose(SourceLoc(), fiddle::Diagnostics::couldNotReadInputFile, inputPath); + return; + } + + // Registering the input file with the `sourceManager` allows us + // to get proper source locations for offsets within it. + // + PathInfo inputPathInfo = PathInfo::makeFromString(inputPath); + SourceFile* inputSourceFile = + sourceManager.createSourceFileWithString(inputPathInfo, inputText); + SourceView* inputSourceView = + sourceManager.createSourceView(inputSourceFile, nullptr, SourceLoc()); + + auto inputFile = RefPtr(new InputFile()); + inputFile->inputFileName = inputFileName; + + // We are going to process the same input file in two different ways: + // + // - We will read the file using the C++-friendly `Lexer` type + // from the Slang `compiler-core` library, in order to scrape + // specially marked C++ declarations and process their contents. + // + // - We will also read the file as plain text, in order to find + // ranges that represent templates to be processed with our + // ad hoc Lua-based template engine. + + // We'll do the token-based parsing step first, and allow it + // to return a `SourceUnit` that we can use to keep track + // of the file. + // + auto sourceUnit = parseSourceUnit(inputSourceView, getOutputFileName(inputFileName)); + + // Then we'll read the same file again looking for template + // lines, and collect that information onto the same + // object, so that we can emit the file back out again, + // potentially with some of its content replaced. + // + auto textTemplateFile = parseTextTemplate(inputSourceView); + + inputFile->scrapedSourceUnit = sourceUnit; + inputFile->textTemplateFile = textTemplateFile; + + inputFiles.add(inputFile); + } + + /// Generate a slug version of the given string. + /// + /// A *slug* is a version of a string that has + /// a visible and obvious dependency on the input + /// text, but that is massaged to conform to the + /// constraints of names for some purpose. + /// + /// In our case, the constraints are to have an + /// identifier that is suitable for use as a + /// preprocessor macro. + /// + String generateSlug(String const& inputText) + { + StringBuilder builder; + int prev = -1; + for (auto c : inputText) + { + // Ordinary alphabetic characters go + // through as-is, but converted to + // upper-case. + // + if (('A' <= c) && (c <= 'Z')) + { + builder.appendChar(c); + } + else if (('a' <= c) && (c <= 'z')) + { + builder.appendChar((c - 'a') + 'A'); + } + else if (('0' <= c) && (c <= '9')) + { + // A digit can be passed through as-is, + // except that we need to account for + // the case where (somehow) the very + // first character is a digit. + if (prev == -1) + builder.appendChar('_'); + builder.appendChar(c); + } + else + { + // We replace any other character with + // an underscore (`_`), but we make + // sure to collapse any sequence of + // consecutive underscores, and to + // ignore characters at the start of + // the string that would turn into + // underscores. + // + if (prev == -1) + continue; + if (prev == '_') + continue; + + c = '_'; + builder.appendChar(c); + } + + prev = c; + } + return builder.produceString(); + } + + void generateAndEmitFilesForInputFile(InputFile* inputFile) + { + // The output file wil name will be the input file + // name, but with the suffix `.fiddle` appended to it. + // + auto inputFileName = inputFile->inputFileName; + String outputFileName = getOutputFileName(inputFileName); + String outputFilePath = options.outputPathPrefix + outputFileName; + + String inputFileSlug = generateSlug(inputFileName); + + // We start the generated file with a header to warn + // people against editing it by hand (not that doing + // so will prevent by-hand edits, but its one of the + // few things we can do). + // + StringBuilder builder; + builder.append("// GENERATED CODE; DO NOT EDIT\n"); + builder.append("//\n"); + + builder.append("// input file: "); + builder.append(inputFile->inputFileName); + builder.append("\n"); + + // There are currently two kinds of generated code + // we need to handle here: + // + // - The code that the scraping tool wants to inject + // at each of the `FIDDLE(...)` macro invocation + // sites. + // + // - The code that is generated from each of the + // `FIDDLE TEMPLATE` constructs. + // + // We will emit both kinds of output to the same + // file, to keep things easy-ish for the client. + + // The first kind of output is the content for + // any `FIDDLE(...)` macro invocations. + // + if (hasAnyFiddleInvocations(inputFile->scrapedSourceUnit)) + { + + builder.append("\n// BEGIN FIDDLE SCRAPER OUTPUT\n"); + builder.append("#ifndef "); + builder.append(inputFileSlug); + builder.append("_INCLUDED\n"); + builder.append("#define "); + builder.append(inputFileSlug); + builder.append("_INCLUDED 1\n"); + builder.append("#ifdef FIDDLE\n"); + builder.append("#undef FIDDLE\n"); + builder.append("#undef FIDDLEX\n"); + builder.append("#undef FIDDLEY\n"); + builder.append("#endif\n"); + builder.append("#define FIDDLEY(ARG) FIDDLE_##ARG\n"); + builder.append("#define FIDDLEX(ARG) FIDDLEY(ARG)\n"); + builder.append("#define FIDDLE FIDDLEX(__LINE__)\n"); + + emitSourceUnitMacros( + inputFile->scrapedSourceUnit, + builder, + &sink, + &sourceManager, + logicalModule); + + builder.append("\n#endif\n"); + builder.append("// END FIDDLE SCRAPER OUTPUT\n"); + } + + if (inputFile->textTemplateFile->textTemplates.getCount() != 0) + { + builder.append("\n// BEGIN FIDDLE TEMPLATE OUTPUT:\n"); + builder.append("#ifdef FIDDLE_GENERATED_OUTPUT_ID\n"); + + generateTextTemplateOutputs( + options.inputPathPrefix + inputFileName, + inputFile->textTemplateFile, + builder, + &sink); + + builder.append("#undef FIDDLE_GENERATED_OUTPUT_ID\n"); + builder.append("#endif\n"); + builder.append("// END FIDDLE TEMPLATE OUTPUT\n"); + } + + builder.append("\n// END OF FIDDLE-GENERATED FILE\n"); + + + { + String outputFileContent = builder.produceString(); + + if (SLANG_FAILED(File::writeAllTextIfChanged( + outputFilePath, + outputFileContent.getUnownedSlice()))) + { + sink.diagnose( + SourceLoc(), + fiddle::Diagnostics::couldNotWriteOutputFile, + outputFilePath); + return; + } + } + + // If we successfully wrote the output file and all of + // its content, it is time to write out new text for + // the *input* file, based on the template file. + // + { + String newInputFileContent = generateModifiedInputFileForTextTemplates( + outputFileName, + inputFile->textTemplateFile, + &sink); + + String inputFilePath = options.inputPathPrefix + inputFileName; + if (SLANG_FAILED(File::writeAllTextIfChanged( + inputFilePath, + newInputFileContent.getUnownedSlice()))) + { + sink.diagnose( + SourceLoc(), + fiddle::Diagnostics::couldNotOverwriteInputFile, + inputFilePath); + return; + } + } + } + + void generateAndEmitFiles() + { + for (auto inputFile : inputFiles) + generateAndEmitFilesForInputFile(inputFile); + } + + void checkModule() { fiddle::checkModule(this->logicalModule, &sink); } + + void execute(int argc, char const* const* argv) + { + // We start by parsing any command-line options + // that were specified. + // + options.parse(sink, argc, argv); + if (sink.getErrorCount()) + return; + + // All of the code that get scraped will be + // organized into a single logical module, + // with no regard for what file each + // declaration came from. + // + logicalModule = new LogicalModule(); + + // We iterate over the input paths specified on + // the command line, to read each in and process + // its text. + // + // This step both scans for declarations that + // are to be scraped, and also reads the any + // template spans. + // + for (auto inputPath : options.inputPaths) + { + processInputFile(inputPath); + } + if (sink.getErrorCount()) + return; + + // In order to build up the data model of the + // scraped declarations (such as what inherits + // from what), we need to perform a minimal + // amount of semantic checking here. + // + checkModule(); + if (sink.getErrorCount()) + return; + + + // Before we go actually running any of the scripts + // that make up the template files, we need to + // put things into the environment that will allow + // those scripts to find the things we've scraped... + // + registerScrapedStuffWithScript(logicalModule); + if (sink.getErrorCount()) + return; + + + // Once we've processed the data model, we + // can generate the code that goes into + // the corresponding output file, as well + // as process any templates in the input + // files. + // + generateAndEmitFiles(); + if (sink.getErrorCount()) + return; + } +}; +} // namespace fiddle + +#define DEBUG_FIDDLE_COMMAND_LINE 0 + +#if DEBUG_FIDDLE_COMMAND_LINE +#include +#endif + +int main(int argc, char const* const* argv) +{ + using namespace fiddle; + using namespace Slang; + + ComPtr writer(new FileWriter(stderr, WriterFlag::AutoFlush)); + + RootNamePool rootNamePool; + + SourceManager sourceManager; + sourceManager.initialize(nullptr, nullptr); + + DiagnosticSink sink(&sourceManager, Lexer::sourceLocationLexer); + sink.writer = writer; + +#if DEBUG_FIDDLE_COMMAND_LINE + fprintf(stderr, "fiddle:"); + for (int i = 1; i < argc; ++i) + { + fprintf(stderr, " %s", argv[i]); + } + fprintf(stderr, "\n"); + + char buffer[1024]; + GetCurrentDirectoryA(sizeof(buffer), buffer); + fprintf(stderr, "cwd: %s\n", buffer); + return 1; +#endif + + try + { + App app(sourceManager, sink, rootNamePool); + app.execute(argc, argv); + } + catch (...) + { + sink.diagnose(SourceLoc(), fiddle::Diagnostics::internalError); + return 1; + } + return 0; +} diff --git a/tools/slang-fiddle/slang-fiddle-options.cpp b/tools/slang-fiddle/slang-fiddle-options.cpp new file mode 100644 index 00000000000..3291dfa63d0 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-options.cpp @@ -0,0 +1,2 @@ +// slang-fiddle-options.cpp +#include "slang-fiddle-options.h" diff --git a/tools/slang-fiddle/slang-fiddle-options.h b/tools/slang-fiddle/slang-fiddle-options.h new file mode 100644 index 00000000000..ae6412ca09c --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-options.h @@ -0,0 +1,61 @@ +// slang-fiddle-options.h +#pragma once + +#include "slang-fiddle-diagnostics.h" + +namespace fiddle +{ +using namespace Slang; + +// + +struct Options +{ +public: + static const char* expectArg(char const* const*& cursor, char const* const* end) + { + if (cursor != end) + return *cursor++; + return nullptr; + } + + void parse(DiagnosticSink& sink, int argc, char const* const* argv) + { + auto argCursor = argv++; + auto argEnd = argCursor + argc; + + if (argCursor != argEnd) + { + appName = *argCursor++; + } + + while (argCursor != argEnd) + { + UnownedTerminatedStringSlice arg = *argCursor++; + if (arg[0] != '-') + { + inputPaths.add(String(arg)); + continue; + } + + if (arg == UnownedTerminatedStringSlice("-i")) + { + inputPathPrefix = expectArg(argCursor, argEnd); + } + else if (arg == UnownedTerminatedStringSlice("-o")) + { + outputPathPrefix = expectArg(argCursor, argEnd); + } + else + { + sink.diagnose(SourceLoc(), Diagnostics::unknownOption, arg); + } + } + } + + String appName = "slang-fiddle"; + String inputPathPrefix = ""; + String outputPathPrefix = ""; + List inputPaths; +}; +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-scrape.cpp b/tools/slang-fiddle/slang-fiddle-scrape.cpp new file mode 100644 index 00000000000..406699d27b0 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-scrape.cpp @@ -0,0 +1,1736 @@ +// slang-fiddle-scrape.cpp +#include "slang-fiddle-scrape.h" + +#include "slang-fiddle-script.h" + +namespace fiddle +{ + +// Parser + +struct Parser +{ +private: + DiagnosticSink& _sink; + List _tokens; + + TokenWithTrivia const* _cursor = nullptr; + TokenWithTrivia const* _end = nullptr; + + LogicalModule* _module = nullptr; + + ContainerDecl* _currentParentDecl = nullptr; + + struct WithParentDecl + { + public: + WithParentDecl(Parser* outer, ContainerDecl* decl) + { + _outer = outer; + _saved = outer->_currentParentDecl; + + outer->_currentParentDecl = decl; + } + + ~WithParentDecl() { _outer->_currentParentDecl = _saved; } + + private: + Parser* _outer; + ContainerDecl* _saved; + }; + +public: + Parser(DiagnosticSink& sink, List const& tokens, LogicalModule* module) + : _sink(sink), _tokens(tokens), _module(module) + { + _cursor = tokens.begin(); + _end = tokens.end() - 1; + } + + bool _isRecovering = false; + + TokenWithTrivia const& peek() { return *_cursor; } + + SourceLoc const& peekLoc() { return peek().getLoc(); } + + TokenType peekType() { return peek().getType(); } + + TokenWithTrivia read() + { + _isRecovering = false; + if (peekType() != TokenType::EndOfFile) + return *_cursor++; + else + return *_cursor; + } + + TokenWithTrivia expect(TokenType expected) + { + if (peekType() == expected) + { + return read(); + } + + if (!_isRecovering) + { + _sink.diagnose(peekLoc(), fiddle::Diagnostics::unexpected, peekType(), expected); + } + else + { + // TODO: need to skip until we see what we expected... + _sink.diagnose(SourceLoc(), fiddle::Diagnostics::internalError); + } + + return TokenWithTrivia(); + } + + TokenWithTrivia expect(const char* expected) + { + if (peekType() == TokenType::Identifier) + { + if (peek().getContent() == expected) + { + return read(); + } + } + + if (!_isRecovering) + { + _sink.diagnose(peekLoc(), fiddle::Diagnostics::unexpected, peekType(), expected); + } + else + { + // TODO: need to skip until we see what we expected... + _sink.diagnose(SourceLoc(), fiddle::Diagnostics::internalError); + } + + return TokenWithTrivia(); + } + + bool advanceIf(TokenType type) + { + if (peekType() == type) + { + read(); + return true; + } + + return false; + } + + bool advanceIf(char const* name) + { + if (peekType() == TokenType::Identifier) + { + if (peek().getContent() == name) + { + read(); + return true; + } + } + + return false; + } + + RefPtr parseCppSimpleExpr() + { + switch (peekType()) + { + case TokenType::Identifier: + { + auto nameToken = expect(TokenType::Identifier); + return new NameExpr(nameToken); + } + break; + + case TokenType::IntegerLiteral: + { + auto token = read(); + return new LiteralExpr(token); + } + break; + + case TokenType::LParent: + { + expect(TokenType::LParent); + auto inner = parseCppExpr(); + expect(TokenType::RParent); + + // TODO: handle a cast, in the case that the lookahead + // implies we should parse one... + switch (peekType()) + { + case TokenType::Identifier: + case TokenType::LParent: + { + auto arg = parseCppExpr(); + return inner; + } + break; + + default: + return inner; + } + } + break; + + default: + expect(TokenType::Identifier); + _sink.diagnose(SourceLoc(), fiddle::Diagnostics::internalError); + return nullptr; + } + } + + RefPtr parseCppExpr() + { + auto base = parseCppSimpleExpr(); + for (;;) + { + switch (peekType()) + { + default: + return base; + + case TokenType::OpMul: + { + expect(TokenType::OpMul); + switch (peekType()) + { + default: + // treat as introducting a pointer type + return base; + } + } + break; + + case TokenType::Scope: + { + expect(TokenType::Scope); + auto memberName = expect(TokenType::Identifier); + base = new StaticMemberRef(base, memberName); + } + break; + case TokenType::LParent: + { + // TODO: actually parse this! + readBalanced(); + } + break; + + case TokenType::OpLess: + { + auto specialize = RefPtr(new SpecializeExpr()); + specialize->base = base; + + // Okay, we have a template application here. + expect(TokenType::OpLess); + specialize->args = parseCppTemplateArgs(); + parseGenericCloser(); + + base = specialize; + } + break; + } + } + } + + RefPtr parseCppSimpleTypeSpecififer() + { + + switch (peekType()) + { + case TokenType::Identifier: + { + auto nameToken = expect(TokenType::Identifier); + return new NameExpr(nameToken); + } + break; + + default: + expect(TokenType::Identifier); + _sink.diagnose(SourceLoc(), fiddle::Diagnostics::internalError); + return nullptr; + } + } + + List> parseCppTemplateArgs() + { + List> args; + for (;;) + { + switch (peekType()) + { + case TokenType::OpGeq: + case TokenType::OpGreater: + case TokenType::OpRsh: + case TokenType::EndOfFile: + return args; + } + + auto arg = parseCppExpr(); + if (arg) + args.add(arg); + + if (!advanceIf(TokenType::Comma)) + return args; + } + } + + void parseGenericCloser() + { + if (advanceIf(TokenType::OpGreater)) + return; + + if (peekType() == TokenType::OpRsh) + { + peek().setType(TokenType::OpGreater); + + return; + } + + expect(TokenType::OpGreater); + } + + RefPtr parseCppTypeSpecifier() + { + auto result = parseCppSimpleTypeSpecififer(); + for (;;) + { + switch (peekType()) + { + default: + return result; + + case TokenType::Scope: + { + expect(TokenType::Scope); + auto memberName = expect(TokenType::Identifier); + auto memberRef = RefPtr(new StaticMemberRef(result, memberName)); + result = memberRef; + } + break; + + case TokenType::OpLess: + { + auto specialize = RefPtr(new SpecializeExpr()); + specialize->base = result; + + // Okay, we have a template application here. + expect(TokenType::OpLess); + specialize->args = parseCppTemplateArgs(); + parseGenericCloser(); + + result = specialize; + } + break; + } + } + } + + struct UnwrappedDeclarator + { + RefPtr type; + TokenWithTrivia nameToken; + }; + + UnwrappedDeclarator unwrapDeclarator(RefPtr declarator, RefPtr type) + { + if (!declarator) + { + UnwrappedDeclarator result; + result.type = type; + return result; + } + + if (auto ptrDeclarator = as(declarator)) + { + return unwrapDeclarator(ptrDeclarator->base, new PtrType(type)); + } + else if (auto nameDeclarator = as(declarator)) + { + UnwrappedDeclarator result; + result.type = type; + result.nameToken = nameDeclarator->nameToken; + return result; + } + else + { + _sink.diagnose(SourceLoc(), Diagnostics::unexpected, "declarator type", "known"); + return UnwrappedDeclarator(); + } + } + + RefPtr parseCppType() + { + auto typeSpecifier = parseCppTypeSpecifier(); + auto declarator = parseCppDeclarator(); + return unwrapDeclarator(declarator, typeSpecifier).type; + } + + RefPtr parseCppBase() + { + // TODO: allow `private` and `protected` + // TODO: insert a default `public` keyword, if one is missing... + advanceIf("public"); + return parseCppType(); + } + + void parseCppAggTypeDecl(RefPtr decl) + { + decl->mode = Mode::Cpp; + + // read the type name + decl->nameToken = expect(TokenType::Identifier); + + // Read the bases clause. + // + // TODO: handle multiple bases... + // + if (advanceIf(TokenType::Colon)) + { + decl->directBaseType = parseCppBase(); + } + + expect(TokenType::LBrace); + addDecl(decl); + WithParentDecl withParent(this, decl); + + // We expect any `FIDDLE()`-marked aggregate type + // declaration to start with a `FIDDLE(...)` invocation, + // so that there is a suitable insertion point for + // the expansion step. + // + { + auto saved = _cursor; + bool found = peekFiddleEllipsisInvocation(); + _cursor = saved; + if (!found) + { + _sink.diagnose( + peekLoc(), + fiddle::Diagnostics::expectedFiddleEllipsisInvocation, + decl->nameToken.getContent()); + } + } + + parseCppDecls(decl); + expect(TokenType::RBrace); + } + + bool peekFiddleEllipsisInvocation() + { + if (!advanceIf("FIDDLE")) + return false; + + if (!advanceIf(TokenType::LParent)) + return false; + + if (!advanceIf(TokenType::Ellipsis)) + return false; + + return true; + } + + RefPtr parseCppSimpleDeclarator() + { + switch (peekType()) + { + case TokenType::Identifier: + { + auto nameToken = expect(TokenType::Identifier); + return RefPtr(new NameDeclarator(nameToken)); + } + + default: + return nullptr; + } + } + + RefPtr parseCppPostfixDeclarator() + { + auto result = parseCppSimpleDeclarator(); + for (;;) + { + switch (peekType()) + { + default: + return result; + + case TokenType::LBracket: + readBalanced(); + return result; + } + } + return result; + } + + RefPtr parseCppDeclarator() + { + advanceIf("const"); + + if (advanceIf(TokenType::OpMul)) + { + auto base = parseCppDeclarator(); + return RefPtr(new PtrDeclarator(base)); + } + else + { + return parseCppPostfixDeclarator(); + } + } + + void parseCppDeclaratorBasedDecl(List> const& fiddleModifiers) + { + auto typeSpecifier = parseCppTypeSpecifier(); + auto declarator = parseCppDeclarator(); + + auto unwrapped = unwrapDeclarator(declarator, typeSpecifier); + + auto varDecl = RefPtr(new VarDecl()); + varDecl->nameToken = unwrapped.nameToken; + varDecl->type = unwrapped.type; + addDecl(varDecl); + + if (advanceIf(TokenType::OpAssign)) + { + varDecl->initExpr = parseCppExpr(); + } + expect(TokenType::Semicolon); + } + + void parseNativeDeclaration(List> const& fiddleModifiers) + { + auto keyword = peek(); + if (advanceIf("namespace")) + { + RefPtr namespaceDecl = new PhysicalNamespaceDecl(); + namespaceDecl->modifiers = fiddleModifiers; + + + // read the namespace name + namespaceDecl->nameToken = expect(TokenType::Identifier); + + expect(TokenType::LBrace); + + addDecl(namespaceDecl); + WithParentDecl withNamespace(this, namespaceDecl); + + parseCppDecls(namespaceDecl); + + expect(TokenType::RBrace); + } + else if (advanceIf("class")) + { + auto decl = RefPtr(new ClassDecl()); + decl->modifiers = fiddleModifiers; + parseCppAggTypeDecl(decl); + } + else if (advanceIf("struct")) + { + auto decl = RefPtr(new StructDecl()); + decl->modifiers = fiddleModifiers; + parseCppAggTypeDecl(decl); + } + else if (peekType() == TokenType::Identifier) + { + // try to parse a declarator-based declaration + // (which for now is probably a field); + // + parseCppDeclaratorBasedDecl(fiddleModifiers); + } + else + { + _sink.diagnose(peekLoc(), fiddle::Diagnostics::unexpected, peekType(), "OTHER"); + _sink.diagnose(SourceLoc(), fiddle::Diagnostics::internalError); + } + } + + List> parseFiddleModifiers() + { + List> modifiers; + + for (;;) + { + switch (peekType()) + { + default: + return modifiers; + + case TokenType::Identifier: + break; + } + + if (advanceIf("abstract")) + { + modifiers.add(new AbstractModifier()); + } + else if (advanceIf("hidden")) + { + modifiers.add(new HiddenModifier()); + } + else + { + return modifiers; + } + } + + return modifiers; + } + + RefPtr parseFiddlePrimaryExpr() + { + switch (peekType()) + { + case TokenType::Identifier: + return new NameExpr(read()); + + case TokenType::LParent: + { + expect(TokenType::LParent); + auto expr = parseFiddleExpr(); + expect(TokenType::RParent); + return expr; + } + + default: + expect(TokenType::Identifier); + return nullptr; + } + } + + List> parseFiddleArgs() + { + List> args; + for (;;) + { + switch (peekType()) + { + case TokenType::RBrace: + case TokenType::RBracket: + case TokenType::RParent: + case TokenType::EndOfFile: + return args; + + default: + break; + } + + auto arg = parseFiddleExpr(); + args.add(arg); + + if (!advanceIf(TokenType::Comma)) + return args; + } + } + + RefPtr parseFiddlePostifxExpr() + { + auto result = parseFiddlePrimaryExpr(); + + for (;;) + { + switch (peekType()) + { + default: + return result; + + case TokenType::Dot: + { + expect(TokenType::Dot); + auto memberName = expect(TokenType::Identifier); + + result = new MemberExpr(result, memberName); + } + break; + + case TokenType::LParent: + { + expect(TokenType::LParent); + auto args = parseFiddleArgs(); + expect(TokenType::RParent); + + result = new CallExpr(result, args); + } + break; + } + } + } + RefPtr parseFiddleExpr() { return parseFiddlePostifxExpr(); } + + RefPtr parseFiddleTypeExpr() { return parseFiddleExpr(); } + + void parseFiddleAggTypeDecl(RefPtr decl) + { + decl->mode = Mode::Fiddle; + + // read the type name + decl->nameToken = expect(TokenType::Identifier); + + // Read the bases clause. + if (advanceIf(TokenType::Colon)) + { + decl->directBaseType = parseFiddleTypeExpr(); + } + + addDecl(decl); + WithParentDecl withParent(this, decl); + + if (advanceIf(TokenType::LBrace)) + { + parseOptionalFiddleModeDecls(); + + expect(TokenType::RBrace); + } + else + { + expect(TokenType::Semicolon); + } + } + + void parseFiddleModeDecl(List> modifiers) + { + if (advanceIf("class")) + { + auto decl = RefPtr(new ClassDecl()); + decl->modifiers = modifiers; + parseFiddleAggTypeDecl(decl); + } + else + { + _sink.diagnose( + peekLoc(), + Diagnostics::unexpected, + peekType(), + "fiddle-mode declaration"); + } + } + + void parseFiddleModeDecl() + { + auto modifiers = parseFiddleModifiers(); + parseFiddleModeDecl(modifiers); + } + + void parseOptionalFiddleModeDecls() + { + for (;;) + { + switch (peekType()) + { + case TokenType::RParent: + case TokenType::RBrace: + case TokenType::RBracket: + case TokenType::EndOfFile: + return; + } + + parseFiddleModeDecl(); + } + } + + void parseFiddleModeDecls(List> modifiers) + { + parseFiddleModeDecl(modifiers); + parseOptionalFiddleModeDecls(); + } + + void parseFiddleNode() + { + auto fiddleToken = expect("FIDDLE"); + + // We will capture the token at this invocation site, + // because later on we will generate a macro that + // this invocation will expand into. + // + auto fiddleMacroInvocation = RefPtr(new FiddleMacroInvocation()); + fiddleMacroInvocation->fiddleToken = fiddleToken; + addDecl(fiddleMacroInvocation); + + // The `FIDDLE` keyword can be followed by parentheses around a bunch of + // fiddle-mode modifiers. + List> fiddleModifiers; + if (advanceIf(TokenType::LParent)) + { + if (advanceIf(TokenType::Ellipsis)) + { + // A `FIDDLE(...)` invocation is a hook for + // our expansion step to insert the generated + // declarations that go into the body of + // the parent declaration. + + fiddleMacroInvocation->node = _currentParentDecl; + + expect(TokenType::RParent); + return; + } + + + // We start off by parsing optional modifiers + fiddleModifiers = parseFiddleModifiers(); + + if (peekType() != TokenType::RParent) + { + // In this case we are expecting a fiddle-mode declaration + // to appear, in which case we will allow any number of full + // fiddle-mode declarations, but won't expect a C++-mode + // declaration to follow. + + // TODO: We should associate these declarations + // as children of the `FiddleMacroInvocation`, + // so that they can be emitted as part of its + // expansion (if we decide to make more use + // of the `FIDDLE()` approach...). + + parseFiddleModeDecls(fiddleModifiers); + expect(TokenType::RParent); + return; + } + expect(TokenType::RParent); + } + else + { + // TODO: diagnose this! + } + + // Any tokens from here on are expected to be in C++-mode + + parseNativeDeclaration(fiddleModifiers); + } + + void addDecl(ContainerDecl* parentDecl, Decl* memberDecl) + { + if (!memberDecl) + return; + + parentDecl->members.add(memberDecl); + + auto physicalParent = as(parentDecl); + if (!physicalParent) + return; + + auto logicalParent = physicalParent->logicalVersion; + if (!logicalParent) + return; + + if (auto physicalNamespace = as(memberDecl)) + { + auto namespaceName = physicalNamespace->nameToken.getContent(); + auto logicalNamespace = findDecl(logicalParent, namespaceName); + if (!logicalNamespace) + { + logicalNamespace = new LogicalNamespace(); + + logicalNamespace->nameToken = physicalNamespace->nameToken; + + logicalParent->members.add(logicalNamespace); + logicalParent->mapNameToMember.add(namespaceName, logicalNamespace); + } + physicalNamespace->logicalVersion = logicalNamespace; + } + else + { + logicalParent->members.add(memberDecl); + } + } + + void addDecl(RefPtr decl) { addDecl(_currentParentDecl, decl); } + + void parseCppDecls(RefPtr parentDecl) + { + for (;;) + { + switch (peekType()) + { + case TokenType::EndOfFile: + case TokenType::RBrace: + case TokenType::RBracket: + case TokenType::RParent: + return; + + default: + break; + } + + parseCppDecl(); + } + } + + void readBalanced() + { + Count skipCount = read().getSkipCount(); + _cursor = _cursor + skipCount; + } + + void parseCppDecl() + { + // We consume raw tokens until we see something + // that ought to start a reflected/extracted declaration. + // + for (;;) + { + switch (peekType()) + { + default: + { + readBalanced(); + continue; + } + + case TokenType::RBrace: + case TokenType::RBracket: + case TokenType::RParent: + case TokenType::EndOfFile: + return; + + case TokenType::Identifier: + break; + + case TokenType::Pound: + // a `#` means we have run into a preprocessor directive + // (or, somehow, we are already *inside* one...). + // + // We don't want to try to intercept anything to do with + // these lines, so we will read until the next end-of-line. + // + read(); + while (!(peek().getToken().flags & TokenFlag::AtStartOfLine)) + { + if (peekType() == TokenType::EndOfFile) + break; + read(); + } + continue; + } + + // Okay, we have an identifier, but is its name + // one that we want to pay attention to? + // + // + auto name = peek().getContent(); + if (name == "FIDDLE") + { + // If the `FIDDLE` is the first token we are seeing, then we will + // start parsing a construct in fiddle-mode: + // + parseFiddleNode(); + } + else + { + // If the name isn't one we recognize, then + // we are just reading raw tokens as usual. + // + readBalanced(); + continue; + } + } + } + + RefPtr parseSourceUnit() + { + RefPtr sourceUnit = new SourceUnit(); + sourceUnit->logicalVersion = _module; + + WithParentDecl withSourceUnit(this, sourceUnit); + while (_cursor != _end) + { + parseCppDecl(); + + switch (peekType()) + { + default: + break; + + case TokenType::RBrace: + case TokenType::RBracket: + case TokenType::RParent: + case TokenType::EndOfFile: + read(); + break; + } + } + read(); + + return sourceUnit; + } +}; + + +// Check + +struct CheckContext +{ +private: + DiagnosticSink& sink; + +public: + CheckContext(DiagnosticSink& sink) + : sink(sink) + { + } + + void checkModule(LogicalModule* module) { checkMemberDecls(module); } + +private: + struct Scope + { + public: + Scope(ContainerDecl* containerDecl, Scope* outer) + : containerDecl(containerDecl), outer(outer) + { + } + + ContainerDecl* containerDecl = nullptr; + Scope* outer = nullptr; + }; + Scope* currentScope = nullptr; + + struct WithScope : Scope + { + WithScope(CheckContext* context, ContainerDecl* containerDecl) + : Scope(containerDecl, context->currentScope) + , _context(context) + , _saved(context->currentScope) + { + context->currentScope = this; + } + + ~WithScope() { _context->currentScope = _saved; } + + private: + CheckContext* _context = nullptr; + Scope* _saved = nullptr; + }; + + + // + void checkDecl(Decl* decl) + { + if (auto aggTypeDecl = as(decl)) + { + checkTypeExprInPlace(aggTypeDecl->directBaseType); + + if (auto baseType = aggTypeDecl->directBaseType) + { + if (auto baseDeclRef = as(baseType)) + { + auto baseDecl = baseDeclRef->decl; + if (auto baseAggTypeDecl = as(baseDecl)) + { + baseAggTypeDecl->directSubTypeDecls.add(aggTypeDecl); + } + } + } + + checkMemberDecls(aggTypeDecl); + } + else if (auto namespaceDecl = as(decl)) + { + checkMemberDecls(namespaceDecl); + } + else if (auto varDecl = as(decl)) + { + // Note: for now we aren't trying to check the type + // or the initial-value expression of a field. + } + else if (as(decl)) + { + } + else + { + sink.diagnose(SourceLoc(), Diagnostics::unexpected, "case in checkDecl", "known type"); + } + } + + void checkMemberDecls(ContainerDecl* containerDecl) + { + WithScope moduleScope(this, containerDecl); + for (auto memberDecl : containerDecl->members) + { + checkDecl(memberDecl); + } + } + + void checkTypeExprInPlace(RefPtr& ioTypeExpr) + { + if (!ioTypeExpr) + return; + ioTypeExpr = checkTypeExpr(ioTypeExpr); + } + + RefPtr checkTypeExpr(Expr* expr) { return checkExpr(expr); } + + RefPtr checkExpr(Expr* expr) + { + if (auto nameExpr = as(expr)) + { + return lookUp(nameExpr->nameToken.getContent()); + } + else + { + sink.diagnose(SourceLoc(), Diagnostics::unexpected, "case in checkExpr", "known type"); + return nullptr; + } + } + + RefPtr lookUp(UnownedStringSlice const& name) + { + for (auto scope = currentScope; scope; scope = scope->outer) + { + auto containerDecl = scope->containerDecl; + // TODO: accelerate lookup with a dictionary on the container... + for (auto memberDecl : containerDecl->members) + { + if (memberDecl->nameToken.getContent() == name) + { + return new DirectDeclRef(memberDecl); + } + } + } + sink.diagnose(SourceLoc(), Diagnostics::undefinedIdentifier, name); + return nullptr; + } +}; + + +// Emit + +struct EmitContext +{ +private: + SourceManager& _sourceManager; + RefPtr _module; + StringBuilder& _builder; + +public: + EmitContext( + StringBuilder& builder, + DiagnosticSink& sink, + SourceManager& sourceManager, + LogicalModule* module) + : _builder(builder), _sourceManager(sourceManager), _module(module) + { + SLANG_UNUSED(sink); + } + + void emitMacrosRec(Decl* decl) + { + emitMacrosForDecl(decl); + if (auto container = as(decl)) + { + for (auto member : container->members) + emitMacrosRec(member); + } + } + +private: + void emitMacrosForDecl(Decl* decl) + { + if (auto fiddleMacroInvocation = as(decl)) + { + emitMacroForFiddleInvocation(fiddleMacroInvocation); + } + else + { + // do nothing with most decls + } + } + + void emitMacroForFiddleInvocation(FiddleMacroInvocation* fiddleInvocation) + { + SourceLoc loc = fiddleInvocation->fiddleToken.getLoc(); + auto humaneLoc = _sourceManager.getHumaneLoc(loc); + auto lineNumber = humaneLoc.line; + +#define MACRO_LINE_ENDING " \\\n" + + // Un-define the old `FIDDLE_#` macro for the + // given line number, since this file might + // be pulling in another generated header + // via one of its dependencies. + // + _builder.append("#ifdef FIDDLE_"); + _builder.append(lineNumber); + _builder.append("\n#undef FIDDLE_"); + _builder.append(lineNumber); + _builder.append("\n#endif\n"); + + _builder.append("#define FIDDLE_"); + _builder.append(lineNumber); + _builder.append("(...)"); + _builder.append(MACRO_LINE_ENDING); + + auto decl = as(fiddleInvocation->node); + if (decl) + { + if (auto base = decl->directBaseType) + { + _builder.append("private: typedef "); + emitTypedDecl(base, "Super"); + _builder.append(";" MACRO_LINE_ENDING); + } + + if (decl->isSubTypeOf("NodeBase")) + { + _builder.append("friend class ::Slang::ASTBuilder;" MACRO_LINE_ENDING); + _builder.append("friend struct ::Slang::SyntaxClassInfo;" MACRO_LINE_ENDING); + + _builder.append("public: static const ::Slang::SyntaxClassInfo " + "kSyntaxClassInfo;" MACRO_LINE_ENDING); + + _builder.append("public: static constexpr ASTNodeType kType = ASTNodeType::"); + _builder.append(decl->nameToken.getContent()); + _builder.append(";" MACRO_LINE_ENDING); + + _builder.append("public: "); + _builder.append(decl->nameToken.getContent()); + _builder.append("() {}" MACRO_LINE_ENDING); + } + _builder.append("public:" MACRO_LINE_ENDING); + } + _builder.append("/* end */\n\n"); + } + + void emitTypedDecl(Expr* expr, const char* name) + { + if (auto declRef = as(expr)) + { + _builder.append(declRef->decl->nameToken.getContent()); + _builder.append(" "); + _builder.append(name); + } + } + +#if 0 + void emitLineDirective(Token const& lexeme) + { + SourceLoc loc = lexeme.getLoc(); + auto humaneLoc = _sourceManager.getHumaneLoc(loc); + _builder.append("\n#line "); + _builder.append(humaneLoc.line); + _builder.append(" \""); + for (auto c : humaneLoc.pathInfo.getName()) + { + if (c == '\\') _builder.append("\\\\"); + else _builder.append(c); + } + _builder.append("\"\n"); + } + + void emitLineDirective(TokenWithTrivia const& token) + { + if (token.getLeadingTrivia().getCount() != 0) + emitLineDirective(token.getLeadingTrivia()[0]); + else + emitLineDirective(token.getToken()); + } + + void emitLineDirective(RawNode* node) + { + emitLineDirective(node->tokens[0]); + } + + + void emitTrivia(List const& trivia) + { + for (auto trivium : trivia) + _builder.append(trivium.getContent()); + } + + void emitRawNode(RawNode* rawNode) + { + for (auto token : rawNode->tokens) + { + emitTrivia(token.getLeadingTrivia()); + _builder.append(token.getContent()); + emitTrivia(token.getTrailingTrivia()); + } + } + + void emitTopLevelNode(Decl* node) + { + if (!node) + return; + + if (node->findModifier()) + return; + + if (auto rawNode = as(node)) + { + // TODO: should emit a `#line` to point back to + // the original source file... + emitLineDirective(rawNode); + + emitRawNode(rawNode); + } + else if (auto decl = as(node)) + { + for (auto child : decl->members) + { + emitTopLevelNode(child); + } + } + else if (auto decl = as(node)) + { + emitExtraMembersForAggTypeDecl(decl); + + for (auto child : decl->members) + { + emitTopLevelNode(child); + } + } + else if (auto varDecl = as(node)) + { + // Note: nothing to be done here... + } + else + { + _sink.diagnose(SourceLoc(), fiddle::Diagnostics::unexpected, "emitTopLevelNode", "unhandled case"); + } + } + + void emitSourceUnit(SourceUnit* sourceUnit) + { + for (auto node : sourceUnit->members) + { + emitTopLevelNode(node); + } + } + + private: +#endif +}; + + +Decl* findDecl_(ContainerDecl* outerDecl, UnownedStringSlice const& name) +{ + for (auto memberDecl : outerDecl->members) + { + if (memberDecl->nameToken.getContent() == name) + return memberDecl; + } + return nullptr; +} + +bool AggTypeDecl::isSubTypeOf(char const* name) +{ + Decl* decl = this; + while (decl) + { + if (decl->nameToken.getContent() == UnownedTerminatedStringSlice(name)) + { + return true; + } + + auto aggType = as(decl); + if (!aggType) + break; + + auto baseTypeExpr = aggType->directBaseType; + if (!baseTypeExpr) + break; + + auto declRef = as(baseTypeExpr); + if (!declRef) + break; + + decl = declRef->decl; + } + return false; +} + +bool isTrivia(TokenType lexemeType) +{ + switch (lexemeType) + { + default: + return false; + + case TokenType::LineComment: + case TokenType::BlockComment: + case TokenType::NewLine: + case TokenType::WhiteSpace: + return true; + } +} + +List collectTokensWithTrivia(TokenList const& lexemes) +{ + TokenReader reader(lexemes); + + List allTokensWithTrivia; + for (;;) + { + RefPtr currentTokenWithTriviaNode = new TokenWithTriviaNode(); + TokenWithTrivia currentTokenWithTrivia = currentTokenWithTriviaNode; + allTokensWithTrivia.add(currentTokenWithTrivia); + + while (isTrivia(reader.peekTokenType())) + { + auto trivia = reader.advanceToken(); + currentTokenWithTriviaNode->leadingTrivia.add(trivia); + } + + auto token = reader.advanceToken(); + currentTokenWithTriviaNode->token = token; + + if (token.type == TokenType::EndOfFile) + return allTokensWithTrivia; + + while (isTrivia(reader.peekTokenType())) + { + auto trivia = reader.advanceToken(); + currentTokenWithTriviaNode->trailingTrivia.add(trivia); + + if (trivia.type == TokenType::NewLine) + break; + } + } +} + +void readTokenTree(List const& tokens, Index& ioIndex); + +void readBalancedToken(List const& tokens, Index& ioIndex, TokenType closeType) +{ + auto open = tokens[ioIndex++]; + auto openNode = (TokenWithTriviaNode*)open; + + Index startIndex = ioIndex; + for (;;) + { + auto token = tokens[ioIndex]; + if (token.getType() == closeType) + { + ioIndex++; + break; + } + + switch (token.getType()) + { + default: + readTokenTree(tokens, ioIndex); + continue; + + case TokenType::RBrace: + case TokenType::RBracket: + case TokenType::RParent: + case TokenType::EndOfFile: + break; + } + break; + } + openNode->skipCount = ioIndex - startIndex; +} + +void readTokenTree(List const& tokens, Index& ioIndex) +{ + switch (tokens[ioIndex].getType()) + { + default: + ioIndex++; + return; + + case TokenType::LBrace: + return readBalancedToken(tokens, ioIndex, TokenType::RBrace); + + case TokenType::LBracket: + return readBalancedToken(tokens, ioIndex, TokenType::RBracket); + + case TokenType::LParent: + return readBalancedToken(tokens, ioIndex, TokenType::RParent); + } +} + +void matchBalancedTokens(List tokens) +{ + Index index = 0; + for (;;) + { + auto& token = tokens[index]; + switch (token.getType()) + { + case TokenType::EndOfFile: + return; + + default: + readTokenTree(tokens, index); + break; + + case TokenType::RBrace: + case TokenType::RBracket: + case TokenType::RParent: + // error!!! + index++; + break; + } + } +} + +bool findOutputFileIncludeDirective(List tokens, String outputFileName) +{ + auto cursor = tokens.begin(); + auto end = tokens.end() - 1; + + while (cursor != end) + { + if (cursor->getType() != TokenType::Pound) + { + cursor++; + continue; + } + cursor++; + + if (cursor->getContent() != "include") + continue; + cursor++; + + if (cursor->getType() != TokenType::StringLiteral) + continue; + + auto includedFileName = getStringLiteralTokenValue(cursor->getToken()); + if (includedFileName == outputFileName) + return true; + } + return false; +} + +RefPtr parseSourceUnit( + SourceView* inputSourceView, + LogicalModule* logicalModule, + RootNamePool* rootNamePool, + DiagnosticSink* sink, + SourceManager* sourceManager, + String outputFileName) +{ + Lexer lexer; + NamePool namePool; + namePool.setRootNamePool(rootNamePool); + + // We suppress any diagnostics that might get emitted during lexing, + // so that we can ignore any files we don't understand. + // + DiagnosticSink lexerSink; + lexer.initialize(inputSourceView, &lexerSink, &namePool, sourceManager->getMemoryArena()); + + auto inputTokens = lexer.lexAllTokens(); + auto tokensWithTrivia = collectTokensWithTrivia(inputTokens); + matchBalancedTokens(tokensWithTrivia); + + Parser parser(*sink, tokensWithTrivia, logicalModule); + auto sourceUnit = parser.parseSourceUnit(); + + // As a quick validation check, if the source file had + // any `FIDDLE()` invocations in it, then we check to + // make sure it also has a `#include` of the corresponding + // output file name... + if (hasAnyFiddleInvocations(sourceUnit)) + { + if (!findOutputFileIncludeDirective(tokensWithTrivia, outputFileName)) + { + sink->diagnose( + inputSourceView->getRange().begin, + fiddle::Diagnostics::expectedIncludeOfOutputHeader, + outputFileName); + } + } + + return sourceUnit; +} + +void push(lua_State* L, Val* val); + +void push(lua_State* L, UnownedStringSlice const& text) +{ + lua_pushlstring(L, text.begin(), text.getLength()); +} + +template +void push(lua_State* L, List const& values) +{ + // Note: Lua tables are naturally indexed starting at 1. + Index nextIndex = 1; + lua_newtable(L); + for (auto value : values) + { + Index index = nextIndex++; + + push(L, value); + lua_seti(L, -2, index); + } +} + +void getAllSubclasses(AggTypeDecl* decl, List>& ioSubclasses) +{ + ioSubclasses.add(decl); + for (auto subclass : decl->directSubTypeDecls) + getAllSubclasses(subclass, ioSubclasses); +} + +List> getAllSubclasses(AggTypeDecl* decl) +{ + List> result; + getAllSubclasses(decl, result); + return result; +} + +int _toStringVal(lua_State* L) +{ + Val* val = (Val*)lua_touserdata(L, 1); + + if (auto directDeclRef = as(val)) + { + val = directDeclRef->decl; + } + + if (auto decl = as(val)) + { + push(L, decl->nameToken.getContent()); + return 1; + } + + lua_pushfstring(L, "fiddle::Val @ 0x%p", val); + return 1; +} + +int _indexVal(lua_State* L) +{ + Val* val = (Val*)lua_touserdata(L, 1); + char const* name = lua_tostring(L, 2); + + if (auto containerDecl = as(val)) + { + for (auto m : containerDecl->members) + { + if (m->nameToken.getContent() == UnownedTerminatedStringSlice(name)) + { + push(L, m); + return 1; + } + } + } + + if (auto classDecl = as(val)) + { + if (strcmp(name, "subclasses") == 0) + { + auto value = getAllSubclasses(classDecl); + push(L, value); + return 1; + } + + if (strcmp(name, "directSuperClass") == 0) + { + push(L, classDecl->directBaseType); + return 1; + } + + if (strcmp(name, "directFields") == 0) + { + List> fields; + for (auto m : classDecl->members) + { + if (auto f = as(m)) + fields.add(f); + } + push(L, fields); + return 1; + } + } + + if (auto decl = as(val)) + { + if (strcmp(name, "isAbstract") == 0) + { + lua_pushboolean(L, decl->findModifier() != nullptr); + return 1; + } + } + + return 0; +} + +void push(lua_State* L, Val* val) +{ + if (!val) + { + lua_pushnil(L); + return; + } + + lua_pushlightuserdata(L, val); + if (luaL_newmetatable(L, "fiddle::Val")) + { + lua_pushcfunction(L, &_indexVal); + lua_setfield(L, -2, "__index"); + + lua_pushcfunction(L, &_toStringVal); + lua_setfield(L, -2, "__tostring"); + } + lua_setmetatable(L, -2); +} + +void registerValWithScript(String name, Val* val) +{ + auto L = getLuaState(); + + push(L, val); + lua_setglobal(L, name.getBuffer()); +} + + +void registerScrapedStuffWithScript(LogicalModule* logicalModule) +{ + for (auto decl : logicalModule->members) + { + if (!decl->nameToken) + continue; + + registerValWithScript(decl->nameToken.getContent(), decl); + } +} + +bool _hasAnyFiddleInvocationsRec(Decl* decl) +{ + if (as(decl)) + return true; + + if (auto container = as(decl)) + { + for (auto m : container->members) + { + if (_hasAnyFiddleInvocationsRec(m)) + return true; + } + } + return false; +} + +bool hasAnyFiddleInvocations(SourceUnit* sourceUnit) +{ + return _hasAnyFiddleInvocationsRec(sourceUnit); +} + +void checkModule(LogicalModule* module, DiagnosticSink* sink) +{ + CheckContext context(*sink); + context.checkModule(module); +} + + +void emitSourceUnitMacros( + SourceUnit* sourceUnit, + StringBuilder& builder, + DiagnosticSink* sink, + SourceManager* sourceManager, + LogicalModule* logicalModule) +{ + // The basic task here is to find each of the + // `FIDDLE()` macro invocations, and for each + // of them produce a matching definition that + // will be used as the expansion of that one + // + + EmitContext context(builder, *sink, *sourceManager, logicalModule); + context.emitMacrosRec(sourceUnit); +} + +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-scrape.h b/tools/slang-fiddle/slang-fiddle-scrape.h new file mode 100644 index 00000000000..5cd0b6d057d --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-scrape.h @@ -0,0 +1,352 @@ +// slang-fiddle-scrape.h +#pragma once + +#include "compiler-core/slang-lexer.h" +#include "slang-fiddle-diagnostics.h" + +namespace fiddle +{ +using namespace Slang; + +class Val : public RefObject +{ +public: +}; + +class Node : public Val +{ +public: +}; + +// Grouping Tokens and Trivia + +class TokenWithTriviaNode : public RefObject +{ +public: + TokenType getType() const { return token.type; } + + List leadingTrivia; + Token token; + List trailingTrivia; + Count skipCount = 0; +}; + +struct TokenWithTrivia +{ +public: + TokenWithTrivia() {} + + TokenWithTrivia(RefPtr node) + : node(node) + { + } + + SourceLoc const& getLoc() const { return node->token.loc; } + + Token const& getToken() const { return node->token; } + + TokenType getType() const { return node ? node->getType() : TokenType::Unknown; } + + UnownedStringSlice getContent() const + { + return node ? node->token.getContent() : UnownedStringSlice(); + } + + Count getSkipCount() const { return node ? node->skipCount : 0; } + + void setType(TokenType type) const { node->token.type = type; } + + List const& getLeadingTrivia() const { return node->leadingTrivia; } + List const& getTrailingTrivia() const { return node->trailingTrivia; } + + operator TokenWithTriviaNode*() { return node; } + +private: + RefPtr node; +}; + + +// Syntax + +class Declarator : public Node +{ +}; + +class NameDeclarator : public Declarator +{ +public: + NameDeclarator(TokenWithTrivia nameToken) + : nameToken(nameToken) + { + } + + TokenWithTrivia nameToken; +}; + +class PtrDeclarator : public Declarator +{ +public: + PtrDeclarator(RefPtr base) + : base(base) + { + } + + RefPtr base; +}; + +class Expr : public Node +{ +public: +}; + +class ModifierNode : public Node +{ +}; + +class AbstractModifier : public ModifierNode +{ +}; +class HiddenModifier : public ModifierNode +{ +}; + +enum class Mode +{ + Fiddle, + Cpp, +}; + +class Decl : public Node +{ +public: + template + T* findModifier() + { + for (auto m : modifiers) + { + if (auto found = as(m)) + return found; + } + return nullptr; + } + + List> modifiers; + TokenWithTrivia nameToken; + Mode mode = Mode::Cpp; +}; + +class ContainerDecl : public Decl +{ +public: + List> members; + Dictionary> mapNameToMember; +}; + +class LogicalContainerDecl : public ContainerDecl +{ +}; + +class LogicalNamespaceBase : public LogicalContainerDecl +{ +}; + +class LogicalModule : public LogicalNamespaceBase +{ +public: +}; + +class LogicalNamespace : public LogicalNamespaceBase +{ +}; + +class PhysicalContainerDecl : public ContainerDecl +{ +public: + LogicalContainerDecl* logicalVersion = nullptr; +}; + +class SourceUnit : public PhysicalContainerDecl +{ +public: +}; + +class PhysicalNamespaceDecl : public PhysicalContainerDecl +{ +public: +}; + +class AggTypeDecl : public ContainerDecl +{ +public: + RefPtr directBaseType; + + List directSubTypeDecls; + + bool isSubTypeOf(char const* name); +}; + +class ClassDecl : public AggTypeDecl +{ +}; +class StructDecl : public AggTypeDecl +{ +}; + +class VarDecl : public Decl +{ +public: + RefPtr type; + RefPtr initExpr; +}; + +class FiddleMacroInvocation : public Decl +{ +public: + TokenWithTrivia fiddleToken; // the actual `FIDDLE` identifier + + RefPtr node; // the node whose generated content should get emitted... +}; + +class UncheckedExpr : public Expr +{ +}; + +class CheckedExpr : public Expr +{ +}; + +class NameExpr : public UncheckedExpr +{ +public: + NameExpr(TokenWithTrivia nameToken) + : nameToken(nameToken) + { + } + + TokenWithTrivia nameToken; +}; + +class LiteralExpr : public UncheckedExpr +{ +public: + LiteralExpr(TokenWithTrivia token) + : token(token) + { + } + + TokenWithTrivia token; +}; + +class MemberExpr : public UncheckedExpr +{ +public: + MemberExpr(RefPtr base, TokenWithTrivia memberNameToken) + : base(base), memberNameToken(memberNameToken) + { + } + + RefPtr base; + TokenWithTrivia memberNameToken; +}; + +class StaticMemberRef : public UncheckedExpr +{ +public: + StaticMemberRef(RefPtr base, TokenWithTrivia memberNameToken) + : base(base), memberNameToken(memberNameToken) + { + } + + RefPtr base; + TokenWithTrivia memberNameToken; +}; + +typedef Expr Arg; + +class PtrType : public UncheckedExpr +{ +public: + PtrType(RefPtr base) + : base(base) + { + } + + RefPtr base; +}; + +class SpecializeExpr : public UncheckedExpr +{ +public: + SpecializeExpr() {} + + RefPtr base; + List> args; +}; + +class CallExpr : public UncheckedExpr +{ +public: + CallExpr(RefPtr base, List> args) + : base(base), args(args) + { + } + + RefPtr base; + List> args; +}; + +class DirectDeclRef : public CheckedExpr +{ +public: + DirectDeclRef(Decl* decl) + : decl(decl) + { + } + + Decl* decl = nullptr; +}; + +// + +Decl* findDecl_(ContainerDecl* outerDecl, UnownedStringSlice const& name); + +template +T* findDecl(ContainerDecl* outerDecl, UnownedStringSlice const& name) +{ + auto decl = findDecl_(outerDecl, name); + if (!decl) + return nullptr; + + auto asType = as(decl); + if (!asType) + { + // TODO: might need this case to be an error... + return nullptr; + } + + return asType; +} + + +RefPtr parseSourceUnit( + SourceView* inputSourceView, + LogicalModule* logicalModule, + RootNamePool* rootNamePool, + DiagnosticSink* sink, + SourceManager* sourceManager, + String outputFileName); + +bool hasAnyFiddleInvocations(SourceUnit* sourceUnit); + +void checkModule(LogicalModule* module, DiagnosticSink* sink); + + +void registerScrapedStuffWithScript(LogicalModule* logicalModule); + +void emitSourceUnitMacros( + SourceUnit* sourceUnit, + StringBuilder& builder, + DiagnosticSink* sink, + SourceManager* sourceManager, + LogicalModule* logicalModule); +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-script.cpp b/tools/slang-fiddle/slang-fiddle-script.cpp new file mode 100644 index 00000000000..5add4e16e38 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-script.cpp @@ -0,0 +1,172 @@ +// slang-fiddle-script.cpp +#include "slang-fiddle-script.h" + +#include "../external/lua/lapi.h" +#include "../external/lua/lauxlib.h" +#include "../external/lua/lualib.h" + +namespace fiddle +{ +DiagnosticSink* _sink = nullptr; +StringBuilder* _builder = nullptr; +Count _templateCounter = 0; + +void diagnoseLuaError(lua_State* L) +{ + size_t size = 0; + char const* buffer = lua_tolstring(L, -1, &size); + String message = UnownedStringSlice(buffer, size); + message = message + "\n"; + if (_sink) + { + _sink->diagnoseRaw(Severity::Error, message.getBuffer()); + } + else + { + fprintf(stderr, "%s", message.getBuffer()); + } +} + +int _handleLuaError(lua_State* L) +{ + diagnoseLuaError(L); + return lua_error(L); +} + +int _original(lua_State* L) +{ + // We ignore the text that we want to just pass + // through unmodified... + return 0; +} + +int _raw(lua_State* L) +{ + size_t size = 0; + char const* buffer = lua_tolstring(L, 1, &size); + + _builder->append(UnownedStringSlice(buffer, size)); + return 0; +} + +int _splice(lua_State* L) +{ + auto savedBuilder = _builder; + + StringBuilder spliceBuilder; + _builder = &spliceBuilder; + + lua_pushvalue(L, 1); + auto result = lua_pcall(L, 0, 1, 0); + + _builder = savedBuilder; + + if (result != LUA_OK) + { + return _handleLuaError(L); + } + + // The actual string value follows whatever + // got printed to the output (unless it is + // nil). + // + _builder->append(spliceBuilder.produceString()); + if (!lua_isnil(L, -1)) + { + size_t size = 0; + char const* buffer = luaL_tolstring(L, -1, &size); + _builder->append(UnownedStringSlice(buffer, size)); + } + return 0; +} + +int _template(lua_State* L) +{ + auto templateID = _templateCounter++; + + _builder->append("\n#if FIDDLE_GENERATED_OUTPUT_ID == "); + _builder->append(templateID); + _builder->append("\n"); + + lua_pushvalue(L, 1); + auto result = lua_pcall(L, 0, 0, 0); + if (result != LUA_OK) + { + return _handleLuaError(L); + } + + _builder->append("\n#endif\n"); + + return 0; +} + +lua_State* L = nullptr; + +void ensureLuaInitialized() +{ + if (L) + return; + + L = luaL_newstate(); + luaL_openlibs(L); + + lua_pushcclosure(L, &_original, 0); + lua_setglobal(L, "ORIGINAL"); + + lua_pushcclosure(L, &_raw, 0); + lua_setglobal(L, "RAW"); + + lua_pushcclosure(L, &_splice, 0); + lua_setglobal(L, "SPLICE"); + + lua_pushcclosure(L, &_template, 0); + lua_setglobal(L, "TEMPLATE"); + + // TODO: register custom stuff here... +} + +lua_State* getLuaState() +{ + ensureLuaInitialized(); + return L; +} + + +String evaluateScriptCode(String originalFileName, String scriptSource, DiagnosticSink* sink) +{ + StringBuilder builder; + _builder = &builder; + _templateCounter = 0; + + ensureLuaInitialized(); + + String luaChunkName = "@" + originalFileName; + + if (LUA_OK != luaL_loadbuffer( + L, + scriptSource.getBuffer(), + scriptSource.getLength(), + luaChunkName.getBuffer())) + { + size_t size = 0; + char const* buffer = lua_tolstring(L, -1, &size); + String message = UnownedStringSlice(buffer, size); + message = message + "\n"; + sink->diagnoseRaw(Severity::Error, message.getBuffer()); + SLANG_ABORT_COMPILATION("fiddle failed during Lua script loading"); + } + + if (LUA_OK != lua_pcall(L, 0, 0, 0)) + { + size_t size = 0; + char const* buffer = lua_tolstring(L, -1, &size); + String message = UnownedStringSlice(buffer, size); + message = message + "\n"; + sink->diagnoseRaw(Severity::Error, message.getBuffer()); + SLANG_ABORT_COMPILATION("fiddle failed during Lua script execution"); + } + + _builder = nullptr; + return builder.produceString(); +} +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-script.h b/tools/slang-fiddle/slang-fiddle-script.h new file mode 100644 index 00000000000..d8221a2e819 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-script.h @@ -0,0 +1,19 @@ +// slang-fiddle-script.h +#pragma once + +#include "../external/lua/lapi.h" +#include "../external/lua/lauxlib.h" +#include "compiler-core/slang-source-loc.h" +#include "core/slang-list.h" +#include "core/slang-string.h" +#include "slang-fiddle-diagnostics.h" +#include "slang-fiddle-scrape.h" + +namespace fiddle +{ +using namespace Slang; + +lua_State* getLuaState(); + +String evaluateScriptCode(String originalFileName, String scriptSource, DiagnosticSink* sink); +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-template.cpp b/tools/slang-fiddle/slang-fiddle-template.cpp new file mode 100644 index 00000000000..c70029d6f48 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-template.cpp @@ -0,0 +1,531 @@ +// slang-fiddle-template.cpp +#include "slang-fiddle-template.h" + +#include "slang-fiddle-script.h" + +namespace fiddle +{ +struct TextTemplateParserBase +{ +protected: + TextTemplateParserBase( + SourceView* inputSourceView, + DiagnosticSink* sink, + UnownedStringSlice source) + : _inputSourceView(inputSourceView) + , _sink(sink) + , _cursor(source.begin()) + , _end(source.end()) + { + } + + SourceView* _inputSourceView = nullptr; + DiagnosticSink* _sink = nullptr; + char const* _cursor = nullptr; + char const* _end = nullptr; + + bool atEnd() { return _cursor == _end; } + + UnownedStringSlice readLine() + { + auto lineBegin = _cursor; + + while (!atEnd()) + { + char const* lineEnd = _cursor; + switch (*_cursor) + { + default: + _cursor++; + continue; + + case '\r': + _cursor++; + if (*_cursor == '\n') + _cursor++; + break; + + case '\n': + _cursor++; + break; + } + + return UnownedStringSlice(lineBegin, lineEnd); + } + + return UnownedStringSlice(lineBegin, _end); + } +}; + +struct TextTemplateParser : TextTemplateParserBase +{ +public: + TextTemplateParser( + SourceView* inputSourceView, + DiagnosticSink* sink, + UnownedStringSlice templateSource) + : TextTemplateParserBase(inputSourceView, sink, templateSource) + { + } + + char const* findScriptStmtLine(UnownedStringSlice line) + { + char const* lineCursor = line.begin(); + char const* lineEnd = line.end(); + while (lineCursor != lineEnd) + { + switch (*lineCursor) + { + default: + return nullptr; + + case ' ': + case '\t': + lineCursor++; + continue; + + case '%': + return lineCursor; + } + } + return nullptr; + } + + List> stmts; + + void addRaw(char const* rawBegin, char const* rawEnd) + { + if (rawBegin == rawEnd) + return; + + auto stmt = RefPtr(new TextTemplateRawStmt()); + stmt->text = UnownedStringSlice(rawBegin, rawEnd); + stmts.add(stmt); + } + + void addScriptStmtLine(char const* sourceBegin, char const* sourceEnd) + { + auto stmt = RefPtr(new TextTemplateScriptStmt()); + stmt->scriptSource = UnownedStringSlice(sourceBegin, sourceEnd); + stmts.add(stmt); + } + + void addScriptSpliceExpr(char const* sourceBegin, char const* sourceEnd) + { + auto stmt = RefPtr(new TextTemplateSpliceStmt()); + stmt->scriptExprSource = UnownedStringSlice(sourceBegin, sourceEnd); + stmts.add(stmt); + } + + bool isIdentifierStartChar(int c) + { + return (('a' <= c) && (c <= 'z')) || (('A' <= c) && (c <= 'Z')) || (c == '_'); + } + + bool isIdentifierChar(int c) { return isIdentifierStartChar(c) || (('0' <= c) && (c <= '9')); } + + RefPtr parseTextTemplateBody() + { + bool isAtStartOfLine = true; + bool isInScriptLine = false; + int depthInSplice = 0; + + char const* currentLineBegin = _cursor; + char const* currentSpanBegin = _cursor; + while (!atEnd()) + { + char const* currentSpanEnd = _cursor; + + bool wasAtStartOfLine = isAtStartOfLine; + isAtStartOfLine = false; + + int c = *_cursor++; + switch (c) + { + default: + break; + + case '\r': + if (*_cursor == '\n') + { + _cursor++; + } + case '\n': + isAtStartOfLine = true; + currentLineBegin = _cursor; + if (isInScriptLine) + { + addScriptStmtLine(currentSpanBegin, currentSpanEnd); + isInScriptLine = false; + currentSpanBegin = currentSpanEnd; + } + break; + + case ' ': + case '\t': + isAtStartOfLine = wasAtStartOfLine; + break; + + case '%': + if (wasAtStartOfLine && !depthInSplice) + { + addRaw(currentSpanBegin, currentLineBegin); + isInScriptLine = true; + currentSpanBegin = _cursor; + } + break; + + case '$': + if (isInScriptLine) + continue; + if (depthInSplice) + SLANG_ABORT_COMPILATION("fiddle encountered a '$' nested inside a splice"); + + if (*_cursor == '(') + { + _cursor++; + addRaw(currentSpanBegin, currentSpanEnd); + depthInSplice = 1; + currentSpanBegin = _cursor; + break; + } + else if (isIdentifierStartChar(*_cursor)) + { + addRaw(currentSpanBegin, currentSpanEnd); + + auto spliceExprBegin = _cursor; + while (isIdentifierChar(*_cursor)) + _cursor++; + auto spliceExprEnd = _cursor; + addScriptSpliceExpr(spliceExprBegin, spliceExprEnd); + currentSpanBegin = _cursor; + break; + } + break; + + case '(': + if (!depthInSplice) + continue; + depthInSplice++; + break; + + case ')': + if (!depthInSplice) + continue; + depthInSplice--; + if (depthInSplice == 0) + { + addScriptSpliceExpr(currentSpanBegin, currentSpanEnd); + currentSpanBegin = _cursor; + } + break; + } + } + addRaw(currentSpanBegin, _end); + + if (stmts.getCount() == 1) + return stmts[0]; + else + { + auto stmt = RefPtr(new TextTemplateSeqStmt()); + stmt->stmts = stmts; + return stmt; + } + } + +private: +}; + + +char const* templateStartMarker = "FIDDLE TEMPLATE"; +char const* outputStartMarker = "FIDDLE OUTPUT"; +char const* endMarker = "FIDDLE END"; + +struct TextTemplateFileParser : TextTemplateParserBase +{ +public: + TextTemplateFileParser(SourceView* inputSourceView, DiagnosticSink* sink) + : TextTemplateParserBase(inputSourceView, sink, inputSourceView->getContent()) + { + } + + RefPtr parseTextTemplateFile() + { + auto textTemplateFile = RefPtr(new TextTemplateFile()); + textTemplateFile->originalFileContent = _inputSourceView->getContent(); + while (!atEnd()) + { + auto textTemplate = parseOptionalTextTemplate(); + if (textTemplate) + textTemplateFile->textTemplates.add(textTemplate); + } + return textTemplateFile; + } + +private: + Count _templateCounter = 0; + + bool matches(UnownedStringSlice const& line, char const* marker) + { + auto index = line.indexOf(UnownedTerminatedStringSlice(marker)); + return index >= 0; + } + + bool findMatchingLine(char const* marker, UnownedStringSlice& outMatchingLine) + { + while (!atEnd()) + { + auto line = readLine(); + if (!matches(line, marker)) + { + // TODO: If the line doesn't match the expected marker, + // but it *does* match one of the other markers, then + // we should consider it a probable error. + + continue; + } + + outMatchingLine = line; + return true; + } + return false; + } + + SourceLoc getLoc(char const* ptr) + { + auto offset = ptr - _inputSourceView->getContent().begin(); + auto startLoc = _inputSourceView->getRange().begin; + auto loc = SourceLoc::fromRaw(startLoc.getRaw() + offset); + return loc; + } + + SourceLoc getLoc(UnownedStringSlice text) { return getLoc(text.begin()); } + + RefPtr parseTextTemplateBody(UnownedStringSlice const& source) + { + TextTemplateParser parser(_inputSourceView, _sink, source); + return parser.parseTextTemplateBody(); + } + + RefPtr parseOptionalTextTemplate() + { + // The idea is pretty simple; we scan through the source, one line at + // a time, until we find a line that matches our template start pattern. + // + // If we *don't* find the start marker, then there must not be any + // templates left. + // + UnownedStringSlice templateStartLine; + if (!findMatchingLine(templateStartMarker, templateStartLine)) + return nullptr; + + char const* templateSourceBegin = _cursor; + + // If we *do* find a start line for a template, then we will expect + // to find the other two kinds of lines, to round things out. + + UnownedStringSlice outputStartLine; + if (!findMatchingLine(outputStartMarker, outputStartLine)) + { + // TODO: need to diagnose a problem here... + _sink->diagnose( + getLoc(templateStartLine), + fiddle::Diagnostics::expectedOutputStartMarker, + outputStartMarker); + } + + char const* templateSourceEnd = outputStartLine.begin(); + + char const* existingOutputBegin = _cursor; + + UnownedStringSlice endLine; + if (!findMatchingLine(endMarker, endLine)) + { + // TODO: need to diagnose a problem here... + _sink->diagnose( + getLoc(templateStartLine), + fiddle::Diagnostics::expectedEndMarker, + endMarker); + } + char const* existingOutputEnd = endLine.begin(); + + auto templateSource = UnownedStringSlice(templateSourceBegin, templateSourceEnd); + auto templateBody = parseTextTemplateBody(templateSource); + + auto textTemplate = RefPtr(new TextTemplate()); + textTemplate->id = _templateCounter++; + textTemplate->templateStartLine = templateStartLine; + textTemplate->templateSource = templateSource; + textTemplate->body = templateBody; + textTemplate->outputStartLine = outputStartLine; + textTemplate->existingOutputContent = + UnownedStringSlice(existingOutputBegin, existingOutputEnd); + textTemplate->endLine = endLine; + return textTemplate; + } +}; + +struct TextTemplateScriptCodeEmitter +{ +public: + TextTemplateScriptCodeEmitter(TextTemplateFile* templateFile) + : _templateFile(templateFile) + { + } + + String emitScriptCodeForTextTemplateFile() + { + // We start by emitting the content of the template + // file out as Lua code, so that we can evaluate + // it all using the Lua VM. + // + // We go to some effort to make sure that the line + // numbers in the generated Lua will match those + // in the input. + // + + char const* originalFileRawSpanStart = _templateFile->originalFileContent.begin(); + for (auto t : _templateFile->textTemplates) + { + flushOriginalFileRawSpan(originalFileRawSpanStart, t->templateSource.begin()); + + evaluateTextTemplate(t); + + originalFileRawSpanStart = t->outputStartLine.begin(); + } + flushOriginalFileRawSpan( + originalFileRawSpanStart, + _templateFile->originalFileContent.end()); + + return _builder.produceString(); + } + +private: + TextTemplateFile* _templateFile = nullptr; + StringBuilder _builder; + + void flushOriginalFileRawSpan(char const* begin, char const* end) + { + if (begin == end) + return; + + // TODO: implement the important stuff... + _builder.append("ORIGINAL [==["); + _builder.append(UnownedStringSlice(begin, end)); + _builder.append("]==]"); + } + + void evaluateTextTemplate(TextTemplate* textTemplate) + { + // TODO: there really needs to be some framing around this... + _builder.append("TEMPLATE(function() "); + evaluateTextTemplateStmt(textTemplate->body); + _builder.append(" end)"); + } + + void evaluateTextTemplateStmt(TextTemplateStmt* stmt) + { + if (auto seqStmt = as(stmt)) + { + for (auto s : seqStmt->stmts) + evaluateTextTemplateStmt(s); + } + else if (auto rawStmt = as(stmt)) + { + _builder.append("RAW [==["); + _builder.append(rawStmt->text); + _builder.append("]==]"); + } + else if (auto scriptStmt = as(stmt)) + { + _builder.append(scriptStmt->scriptSource); + _builder.append(" "); + } + else if (auto spliceStmt = as(stmt)) + { + _builder.append("SPLICE(function()return("); + _builder.append(spliceStmt->scriptExprSource); + _builder.append(")end)"); + } + else + { + SLANG_ABORT_COMPILATION( + "fiddle encountered an unknown construct when converting a text template to Lua"); + } + } +}; + + +RefPtr parseTextTemplateFile(SourceView* inputSourceView, DiagnosticSink* sink) +{ + TextTemplateFileParser parser(inputSourceView, sink); + return parser.parseTextTemplateFile(); +} + +void generateTextTemplateOutputs( + String originalFileName, + TextTemplateFile* file, + StringBuilder& builder, + DiagnosticSink* sink) +{ + TextTemplateScriptCodeEmitter emitter(file); + String scriptCode = emitter.emitScriptCodeForTextTemplateFile(); + + String output = evaluateScriptCode(originalFileName, scriptCode, sink); + + builder.append(output); + builder.append("\n"); +} + +String generateModifiedInputFileForTextTemplates( + String templateOutputFileName, + TextTemplateFile* file, + DiagnosticSink* sink) +{ + // The basic idea here is that we need to emit most of + // the body of the file exactly as it originally + // appeared, and then only modifify the few lines + // that represent the text template output. + // + // TODO(tfoley): We could also use this as an opportunity + // to insert the `FIDDLE(...)` markers that the scraping + // tool needs, but that is more work than makes sense + // right now. + + StringBuilder builder; + + + char const* originalFileRawSpanStart = file->originalFileContent.begin(); + for (auto t : file->textTemplates) + { + builder.append( + UnownedStringSlice(originalFileRawSpanStart, t->existingOutputContent.begin())); + + builder.append("#define FIDDLE_GENERATED_OUTPUT_ID "); + builder.append(t->id); + builder.append("\n"); + builder.append("#include \""); + for (auto c : templateOutputFileName) + { + switch (c) + { + case '"': + case '\\': + builder.appendChar('\\'); + builder.appendChar(c); + break; + + default: + builder.appendChar(c); + break; + } + } + builder.append("\"\n"); + originalFileRawSpanStart = t->existingOutputContent.end(); + } + builder.append(UnownedStringSlice(originalFileRawSpanStart, file->originalFileContent.end())); + + return builder.produceString(); +} + +} // namespace fiddle diff --git a/tools/slang-fiddle/slang-fiddle-template.h b/tools/slang-fiddle/slang-fiddle-template.h new file mode 100644 index 00000000000..51798c902a5 --- /dev/null +++ b/tools/slang-fiddle/slang-fiddle-template.h @@ -0,0 +1,81 @@ +// slang-fiddle-template.h +#pragma once + +#include "compiler-core/slang-source-loc.h" +#include "core/slang-list.h" +#include "core/slang-string.h" +#include "slang-fiddle-diagnostics.h" + +namespace fiddle +{ +using namespace Slang; + +class TextTemplateStmt : public RefObject +{ +public: +}; + +class TextTemplateScriptStmt : public TextTemplateStmt +{ +public: + UnownedStringSlice scriptSource; +}; + +class TextTemplateRawStmt : public TextTemplateStmt +{ +public: + // TODO(tfoley): Add a `SourceLoc` here, so + // that we can emit approriate `#line` directives + // to the output... + + UnownedStringSlice text; +}; + +class TextTemplateSpliceStmt : public TextTemplateStmt +{ +public: + UnownedStringSlice scriptExprSource; +}; + +class TextTemplateSeqStmt : public TextTemplateStmt +{ +public: + List> stmts; +}; + +class TextTemplate : public RefObject +{ +public: + /// ID of this template within the enclosing file + Index id; + + UnownedStringSlice templateStartLine; + UnownedStringSlice outputStartLine; + UnownedStringSlice endLine; + + UnownedStringSlice templateSource; + UnownedStringSlice existingOutputContent; + + RefPtr body; +}; + +class TextTemplateFile : public RefObject +{ +public: + UnownedStringSlice originalFileContent; + List> textTemplates; +}; + +RefPtr parseTextTemplateFile(SourceView* inputSourceView, DiagnosticSink* sink); + +void generateTextTemplateOutputs( + String originalFileName, + TextTemplateFile* file, + StringBuilder& builder, + DiagnosticSink* sink); + +String generateModifiedInputFileForTextTemplates( + String templateOutputFileName, + TextTemplateFile* file, + DiagnosticSink* sink); +} // namespace fiddle