Add external import path switch

Author: rikkimax

compiler/src/dmd/cli.d

@@ -337,11 +337,12 @@ dmd -cov -unittest myprog.d
             (only imports).`,
         ),
         Option("dllimport=<value>",
-            "Windows only: select symbols to dllimport (none/defaultLibsOnly/all)",
-            `Which global variables to dllimport implicitly if not defined in a root module
+            "Windows only: select symbols to dllimport (none/defaultLibsOnly/externalOnly/all)",
+            `Which symbols to dllimport implicitly if not defined in a module that is being compiled
             $(UL
                 $(LI $(I none): None)
-                $(LI $(I defaultLibsOnly): Only druntime/Phobos symbols)
+                $(LI $(I defaultLibsOnly): Only druntime/Phobos symbols and any from a module that is marked as external to binary)
+                $(LI $(I externalOnly): Only symbols found from a module that is marked as external to binary)
                 $(LI $(I all): All)
             )`,
         ),
@@ -460,6 +461,9 @@ dmd -cov -unittest myprog.d
         Option("I=<directory>",
             "look for imports also in directory"
         ),
+        Option("extI=<directory>",
+            "look for imports that are out of the currently compiling binary, used to set the module as DllImport"
+        ),
         Option("i[=<pattern>]",
             "include imported modules in the compilation",
             q"{$(P Enables "include imports" mode, where the compiler will include imported

compiler/src/dmd/dmdparams.d

@@ -34,6 +34,7 @@ enum SymImport : ubyte
 {
     none,               /// no symbols
     defaultLibsOnly,    /// only druntime/phobos symbols
+    externalOnly,       /// any module that is known to be external to binary (-extI)
     all,                /// all non-root symbols
 }
 

compiler/src/dmd/dmodule.d

@@ -417,6 +417,8 @@ extern (C++) final class Module : Package
     SearchOptFlags searchCacheFlags;       // cached flags
     bool insearch;
 
+    bool isExplicitlyOutOfBinary; // Is this module known to be out of binary, and must be DllImport'd?
+
     /**
      * A root module is one that will be compiled all the way to
      * object code.  This field holds the root module that caused
@@ -536,6 +538,7 @@ extern (C++) final class Module : Package
         auto m = new Module(loc, filename, ident, 0, 0);
 
         // TODO: apply import path information (pathInfo) on to module
+        m.isExplicitlyOutOfBinary = importPathThatFindUs.isOutOfBinary;
 
         if (!m.read(loc))
             return null;

compiler/src/dmd/e2ir.d

@@ -498,52 +498,114 @@ private __gshared StringTable!(Symbol*) *stringTab;
 /*********************************************
  * Figure out whether a data symbol should be dllimported
  * Params:
- *      var = declaration of the symbol
+ *      symbl = declaration of the symbol
  * Returns:
  *      true if symbol should be imported from a DLL
  */
-bool isDllImported(Dsymbol var)
+bool isDllImported(Dsymbol symbl)
 {
+    // Windows is the only platform which dmd supports, that uses the DllImport/DllExport scheme.
     if (!(target.os & Target.OS.Windows))
         return false;
-    if (var.isImportedSymbol())
+
+    // If function does not have a body, check to see if its marked as DllImport or is set to be exported.
+    // If a global variable has both export + extern, it is DllImport
+    if (symbl.isImportedSymbol())
         return true;
-    if (driverParams.symImport == SymImport.none)
-        return false;
-    if (auto vd = var.isDeclaration())
+
+    // Functions can go through the generated trampoline function.
+    // Not efficient, but it works.
+    if (symbl.isFuncDeclaration())
+        return false; // can always jump through import table
+
+    // Global variables are allowed, but not TLS or read only memory.
+    if (auto vd = symbl.isDeclaration())
+    {
         if (!vd.isDataseg() || vd.isThreadlocal())
             return false;
-    if (var.isFuncDeclaration())
-        return false; // can always jump through import table
-    if (auto tid = var.isTypeInfoDeclaration())
+    }
+
+    final switch(driverParams.symImport)
+    {
+        case SymImport.none:
+            // If DllImport overriding is disabled, do not change dllimport status.
+            return false;
+
+        case SymImport.externalOnly:
+            // Only modules that are marked as out of binary will be DllImport
+            break;
+
+        case SymImport.defaultLibsOnly:
+        case SymImport.all:
+            // If to access anything in druntime/phobos you need DllImport, verify against this.
+            break;
+    }
+    const systemLibraryNeedDllImport = driverParams.symImport != SymImport.externalOnly;
+
+    // For TypeInfo's check to see if its in druntime and DllImport it
+    if (auto tid = symbl.isTypeInfoDeclaration())
     {
+        // Built in TypeInfo's are defined in druntime
         if (builtinTypeInfo(tid.tinfo))
-            return true;
+            return systemLibraryNeedDllImport;
+
+        // Convert TypeInfo to its symbol
         if (auto ad = isAggregate(tid.type))
-            var = ad;
+            symbl = ad;
     }
-    if (driverParams.symImport == SymImport.defaultLibsOnly)
+
     {
-        auto m = var.getModule();
+        // Filter the symbol based upon the module it is in.
+
+        auto m = symbl.getModule();
         if (!m || !m.md)
             return false;
-        const id = m.md.packages.length ? m.md.packages[0] : null;
-        if (id && id != Id.core && id != Id.std)
-            return false;
-        if (!id && m.md.id != Id.std && m.md.id != Id.object)
+
+        if (driverParams.symImport == SymImport.all || m.isExplicitlyOutOfBinary)
+        {
+            // If a module is specified as being out of binary (-extI), then it is allowed to be DllImport.
+        }
+        else if (driverParams.symImport == SymImport.externalOnly)
+        {
+            // Module is in binary, therefore not DllImport
             return false;
+        }
+        else if (systemLibraryNeedDllImport)
+        {
+            // Filter out all modules that are not in druntime/phobos if we are only doing default libs only
+
+            const id = m.md.packages.length ? m.md.packages[0] : null;
+            if (id && id != Id.core && id != Id.std)
+                return false;
+            if (!id && m.md.id != Id.std && m.md.id != Id.object)
+                return false;
+        }
     }
-    else if (driverParams.symImport != SymImport.all)
-        return false;
-    if (auto mod = var.isModule())
-        return !mod.isRoot(); // non-root ModuleInfo symbol
-    if (var.inNonRoot())
+
+    // If symbol is a ModuleInfo, check to see if module is being compiled.
+    if (auto mod = symbl.isModule())
+    {
+        const isBeingCompiled = mod.isRoot();
+        return !isBeingCompiled; // non-root ModuleInfo symbol
+    }
+
+    // Check to see if a template has been instatiated in current compilation,
+    //  if it is defined in a external module, its DllImport.
+    if (symbl.inNonRoot())
         return true; // not instantiated, and defined in non-root
-    if (auto ti = var.isInstantiated()) // && !defineOnDeclare(sym, false))
+
+    // If a template has been instatiated, only DllImport if it is codegen'ing
+    if (auto ti = symbl.isInstantiated()) // && !defineOnDeclare(sym, false))
         return !ti.needsCodegen(); // instantiated but potentially culled (needsCodegen())
-    if (auto vd = var.isVarDeclaration())
+
+    // If a variable declaration and is extern
+    if (auto vd = symbl.isVarDeclaration())
+    {
+        // Shouldn't this be including an export check too???
         if (vd.storage_class & STC.extern_)
             return true; // externally defined global variable
+    }
+
     return false;
 }
 

compiler/src/dmd/frontend.h

@@ -7111,6 +7111,7 @@ class Module final : public Package
     Dsymbol* searchCacheSymbol;
     uint32_t searchCacheFlags;
     bool insearch;
+    bool isExplicitlyOutOfBinary;
     Module* importedFrom;
     Array<Dsymbol* >* decldefs;
     Array<Module* > aimports;
@@ -8349,12 +8350,15 @@ struct Verbose final
 struct ImportPathInfo final
 {
     const char* path;
+    bool isOutOfBinary;
     ImportPathInfo() :
-        path()
+        path(),
+        isOutOfBinary()
     {
     }
-    ImportPathInfo(const char* path) :
-        path(path)
+    ImportPathInfo(const char* path, bool isOutOfBinary = false) :
+        path(path),
+        isOutOfBinary(isOutOfBinary)
         {}
 };
 

compiler/src/dmd/globals.d

@@ -153,6 +153,7 @@ extern(C++) struct Verbose
 
 extern (C++) struct ImportPathInfo {
     const(char)* path; // char*'s of where to look for import modules
+    bool isOutOfBinary; // Will any module found from this path be out of binary?
 }
 
 /// Put command line switches in here

compiler/src/dmd/globals.h

@@ -158,9 +158,17 @@ struct Verbose
 struct ImportPathInfo
 {
     const char* path;
+    d_bool isOutOfBinary;
 
-    ImportPathInfo() : path(NULL) { }
-    ImportPathInfo(const char* p) : path(p) { }
+    ImportPathInfo() :
+        path(),
+        isOutOfBinary()
+    {
+    }
+    ImportPathInfo(const char* path, d_bool isOutOfBinary = false) :
+        path(path),
+        isOutOfBinary(isOutOfBinary)
+        {}
 };
 
 // Put command line switches in here

compiler/src/dmd/mars.d

@@ -829,11 +829,14 @@ bool parseCommandLine(const ref Strings arguments, const size_t argc, ref Param
                 case "defaultLibsOnly":
                     driverParams.symImport = SymImport.defaultLibsOnly;
                     break;
+                case "externalOnly":
+                    driverParams.symImport = SymImport.externalOnly;
+                    break;
                 case "all":
                     driverParams.symImport = SymImport.all;
                     break;
                 default:
-                    error("unknown dllimport '%.*s', must be 'none', 'defaultLibsOnly' or 'all'", cast(int) imp.length, imp.ptr);
+                    error("unknown dllimport '%.*s', must be 'none', 'defaultLibsOnly', 'externalOnly' or 'all'", cast(int) imp.length, imp.ptr);
             }
         }
         else if (arg == "-fIBT")
@@ -1573,6 +1576,13 @@ bool parseCommandLine(const ref Strings arguments, const size_t argc, ref Param
         {
             params.imppath.push(ImportPathInfo(p + 2 + (p[2] == '=')));
         }
+        else if (startsWith(p + 1, "extI"))
+        {
+            // External import path switch -extI
+            auto importPathInfo = ImportPathInfo(p + 5 + (p[5] == '='));
+            importPathInfo.isOutOfBinary = true;
+            params.imppath.push(importPathInfo);
+        }
         else if (p[1] == 'm' && p[2] == 'v' && p[3] == '=') // https://dlang.org/dmd.html#switch-mv
         {
             if (p[4] && strchr(p + 5, '='))

compiler/src/dmd/module.h

@@ -100,6 +100,8 @@ class Module final : public Package
     SearchOptFlags searchCacheFlags;       // cached flags
     d_bool insearch;
 
+    d_bool isExplicitlyOutOfBinary; // Is this module known to be out of binary, and must be DllImport'd?
+
     // module from command line we're imported from,
     // i.e. a module that will be taken all the
     // way to an object file

compiler/test/dshell/dll_cxx.d

@@ -44,7 +44,7 @@ int main()
     // The arguments have to be passed as an array, because run would replace '/' with '\\' otherwise.
     run(dllCmd);
 
-    run(`$DMD -m$MODEL -g -od=$OUTPUT_BASE -of=$EXE_NAME $SRC/testdll.d ` ~ mainExtra);
+    run(`$DMD -m$MODEL -g -od=$OUTPUT_BASE -of=$EXE_NAME -extI=$EXTRA_FILES${SEP}dll_cxx${SEP}external $SRC/testdll.d ` ~ mainExtra);
 
     run(`$EXE_NAME`, stdout, stderr, [`LD_LIBRARY_PATH`: Vars.OUTPUT_BASE]);
 

compiler/test/dshell/extra-files/dll_cxx/external/binding.d

@@ -0,0 +1,3 @@
+module binding;
+
+extern (C) __gshared extern int testExternalImportVar;

compiler/test/dshell/extra-files/dll_cxx/mydll.cpp

@@ -73,3 +73,9 @@ EXPORT int getSomeValueCPP19660(void)
 {
     return someValueCPP19660;
 }
+
+extern "C"
+{
+    // tests -extI switch for variables
+    EXPORT int testExternalImportVar = 0xF1234;
+}

compiler/test/dshell/extra-files/dll_cxx/testdll.d

@@ -107,4 +107,7 @@ void main()
 {
     test22323();
     test19660();
+
+    import binding;
+    assert(testExternalImportVar == 0xF1234);
 }