Merge remote-tracking branch 'upstream/master' into stable

Author: dkorpel

.github/workflows/main.yml

@@ -52,7 +52,7 @@ jobs:
       fail-fast: false
       matrix:
         # Latest stable version, update at will
-        os: [ macOS-12, ubuntu-20.04, windows-2019 ]
+        os: [ macOS-13, ubuntu-22.04, windows-2019 ]
         dc:
           # Always test latest as that is what we use to compile on release
           - dmd-latest
@@ -65,12 +65,21 @@ jobs:
           - dmd-2.099.1
           - dmd-2.102.2
           - dmd-2.105.3
+          - dmd-2.108.1
         include:
           - { do_test: false }
           - { dc: dmd-latest, do_test: true }
           - { dc: ldc-latest, do_test: true }
           - { dc: dmd-master, do_test: true }
           - { dc: ldc-master, do_test: true }
+          # Test on ARM64
+          - { os: macOS-14, dc: ldc-latest, do_test: true }
+        exclude:
+          # Error with those versions:
+          # ld: multiple errors: symbol count from symbol table and dynamic symbol table differ in [.../dub.o]; address=0x0 points to section(2) with no content in '[...]/osx/lib/libphobos2.a[3177](config_a68_4c3.o)'
+          - { os: macOS-13, dc: dmd-2.099.1 }
+          - { os: macOS-13, dc: dmd-2.102.2 }
+          - { os: macOS-13, dc: dmd-2.105.3 }
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -79,7 +88,9 @@ jobs:
     - name: '[OSX] Install dependencies'
       if: runner.os == 'macOS'
       run: |
-        brew install pkg-config coreutils
+        # We need to install GNU utils as the test-suite scripts expect it.
+        # Without them we may get slightly different behavior in tests and hard-to-track failures
+        brew install coreutils diffutils
         echo "PKG_CONFIG_PATH=/usr/local/opt/[email protected]/lib/pkgconfig/" >> $GITHUB_ENV
 
     - name: '[Linux] Install dependencies'

.github/workflows/pr_info_untrusted.yml

@@ -20,7 +20,7 @@ jobs:
     - name: '[Linux] Install dependencies'
       if: runner.os == 'Linux'
       run: |
-        sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev netcat
+        sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
 
     # Compiler to test with
     - name: Prepare compiler

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ macOS-12, ubuntu-20.04, windows-2019 ]
+        os: [ macOS-13, ubuntu-22.04, windows-2019 ]
         arch: [ x86_64 ]
         include:
           - { os: windows-2019, arch: i686 }

changelog/fix-cimport-paths.dd

@@ -0,0 +1,3 @@
+Fix issue where cImportPaths wasn't working with dmd and ldc
+
+dub was passing -I<import path> instead of -P-I<import path> as is required by those compilers

dub.sdl

@@ -7,10 +7,6 @@ license "MIT"
 
 targetPath "bin"
 
-dflags "-preview=in" platform="dmd"
-dflags "-preview=in" platform="ldc"
-//Disabled due to ICEs in gdc.
-//dflags "-fpreview=in" platform="gdc"
 // Deprecated module(s)
 excludedSourceFiles "source/dub/packagesupplier.d"
 
@@ -21,6 +17,11 @@ configuration "application" {
 	// Uncomment to get rich output about the file parsing and json <-> YAML
 	// integrity checks
 	//debugVersions "ConfigFillerDebug"
+
+	dflags "-preview=in" platform="dmd"
+	dflags "-preview=in" platform="ldc"
+	//Disabled due to ICEs in gdc.
+	//dflags "-fpreview=in" platform="gdc"
 }
 
 configuration "library" {
@@ -31,13 +32,13 @@ configuration "library" {
 }
 
 configuration "library-nonet" {
-	dependency "vibe-d:http" version=">=0.9.0 <0.10.0" optional=true
+	dependency "vibe-d:http" version=">=0.9.0 <0.11.0" optional=true
 	targetType "library"
 	excludedSourceFiles "source/app.d"
 }
 
 configuration "dynamic-library-nonet" {
-	dependency "vibe-d:http" version=">=0.9.0 <0.10.0" optional=true
+	dependency "vibe-d:http" version=">=0.9.0 <0.11.0" optional=true
 	targetType "dynamicLibrary"
 	excludedSourceFiles "source/app.d"
 }

dub.selections.json

@@ -1,21 +1,19 @@
 {
 	"fileVersion": 1,
 	"versions": {
-		"botan": "1.12.19",
-		"botan-math": "1.0.3",
-		"diet-ng": "1.8.1",
-		"eventcore": "0.9.30",
-		"libasync": "0.8.6",
-		"libev": "5.0.0+4.04",
-		"libevent": "2.0.2+2.0.16",
-		"memutils": "1.0.10",
-		"mir-linux-kernel": "1.0.1",
-		"openssl": "3.3.3",
+		"diet-ng": "1.8.2",
+		"eventcore": "0.9.35",
+		"mir-linux-kernel": "1.2.1",
+		"openssl": "3.3.4",
 		"openssl-static": "1.0.5+3.0.8",
 		"stdx-allocator": "2.77.5",
 		"taggedalgebraic": "0.11.23",
-		"vibe-container": "1.3.1",
-		"vibe-core": "2.8.5",
-		"vibe-d": "0.9.8"
+		"vibe-container": "1.4.0",
+		"vibe-core": "2.9.6",
+		"vibe-d": "0.10.1",
+		"vibe-http": "1.2.1",
+		"vibe-inet": "1.1.0",
+		"vibe-serialization": "1.0.7",
+		"vibe-stream": "1.1.1"
 	}
 }

source/dub/compilers/compiler.d

@@ -171,20 +171,24 @@ interface Compiler {
 			format("%s failed with exit code %s.", args[0], status));
 	}
 
+	/** Default compiler arguments for performing a probe. They should
+		be the D compiler equivalent of "don't output executables"
+	*/
+	protected string[] defaultProbeArgs() const;
+
 	/** Compiles platform probe file with the specified compiler and parses its output.
 		Params:
 			compiler_binary =	binary to invoke compiler with
-			args			=	arguments for the probe compilation
-			arch_override	=	special handler for x86_mscoff
+			arch_flags      =   compiler specific flags derived from the user's arch override
 	*/
-	protected final BuildPlatform probePlatform(string compiler_binary, string[] args, string arch_override)
+	protected final BuildPlatform probePlatform(string compiler_binary, string[] arch_flags)
 	{
 		import dub.compilers.utils : generatePlatformProbeFile, readPlatformSDLProbe;
 		import std.string : format, strip;
 
-		NativePath fil = generatePlatformProbeFile();
+		immutable fileArg = generatePlatformProbeFile().toNativeString;
 
-		auto result = execute(compiler_binary ~ args ~ fil.toNativeString());
+		auto result = execute(compiler_binary ~ defaultProbeArgs ~ arch_flags ~ fileArg);
 		enforce!CompilerInvocationException(result.status == 0,
 				format("Failed to invoke the compiler %s to determine the build platform: %s",
 				compiler_binary, result.output));
@@ -200,22 +204,20 @@ interface Compiler {
 			build_platform.compilerVersion = ver;
 		}
 
-		// Skip the following check for LDC, emitting a warning if the specified `-arch`
-		// cmdline option does not lead to the same string being found among
-		// `build_platform.architecture`, as it's brittle and doesn't work with triples.
-		if (build_platform.compiler != "ldc") {
-			if (arch_override.length && !build_platform.architecture.canFind(arch_override) &&
-				!(build_platform.compiler == "dmd" && arch_override.among("x86_omf", "x86_mscoff")) // Will be fixed in determinePlatform
-			) {
-				logWarn(`Failed to apply the selected architecture %s. Got %s.`,
-					arch_override, build_platform.architecture);
-			}
-		}
-
 		return build_platform;
 	}
+
 }
 
 private {
 	Compiler[] s_compilers;
 }
+
+/// Adds the given flags to the build settings if desired, otherwise informs the user
+package void maybeAddArchFlags(ref BuildSettings settings, bool keep_arch, string[] arch_flags, string arch_override) {
+	if (keep_arch)
+		settings.addDFlags(arch_flags);
+	else if (arch_override.length) {
+		logDebug("Ignoring arch_override '%s' for better caching because it doesn't affect the build", arch_override);
+	}
+}

source/dub/compilers/dmd.d

@@ -110,34 +110,37 @@ config    /etc/dmd.conf
 	{
 		// Set basic arch flags for the probe - might be revised based on the exact value + compiler version
 		string[] arch_flags;
-		if (arch_override.length)
-			arch_flags = [ arch_override != "x86_64" ? "-m32" : "-m64" ];
-		else
-		{
-			// Don't use Optlink by default on Windows
-			version (Windows) {
-				const is64bit = isWow64();
-				if (!is64bit.isNull)
-					arch_flags = [ is64bit.get ? "-m64" : "-m32" ];
-			}
-		}
-
-		BuildPlatform bp = probePlatform(
-			compiler_binary,
-			arch_flags ~ ["-quiet", "-c", "-o-", "-v"],
-			arch_override
-		);
-
 		switch (arch_override) {
 			default: throw new UnsupportedArchitectureException(arch_override);
-			case "": break;
+			case "":
+				// Don't use Optlink by default on Windows
+				version (Windows) {
+					const is64bit = isWow64();
+					if (!is64bit.isNull)
+						arch_flags = [ is64bit.get ? "-m64" : "-m32" ];
+				}
+				break;
 			// DMD 2.099  made MsCOFF the default, and DMD v2.109 removed OMF
 			// support. Default everything to MsCOFF, people wanting to use OMF
 			// should use an older DMD / dub.
 			case "x86", "x86_omf", "x86_mscoff": arch_flags = ["-m32"]; break;
 			case "x86_64": arch_flags = ["-m64"]; break;
 		}
-		settings.addDFlags(arch_flags);
+
+		auto bp = probePlatform(compiler_binary, arch_flags);
+
+		bool keep_arch;
+		if (arch_flags.length)
+			keep_arch = bp.architecture != probePlatform(compiler_binary, []).architecture;
+		settings.maybeAddArchFlags(keep_arch, arch_flags, arch_override);
+
+		if (arch_override.length
+			&& !bp.architecture.canFind(arch_override)
+			&& !arch_override.among("x86_omf", "x86_mscoff")
+			) {
+			logWarn(`Failed to apply the selected architecture %s. Got %s.`,
+					arch_override, bp.architecture);
+		}
 
 		return bp;
 	}
@@ -171,10 +174,20 @@ config    /etc/dmd.conf
 	version (LDC) unittest {
 		import std.conv : to;
 
+		version (ARM)
+			enum isARM = true;
+		version (AArch64)
+			enum isARM = true;
+		else
+			enum isARM = false;
+
 		BuildSettings settings;
 		auto compiler = new DMDCompiler;
 		auto bp = compiler.determinePlatform(settings, "ldmd2", "x86");
-		assert(bp.architecture.canFind("x86"), bp.architecture.to!string);
+		static if (isARM)
+			assert(bp.architecture.canFind("arm"), bp.architecture.to!string);
+		else
+			assert(bp.architecture.canFind("x86"), bp.architecture.to!string);
 		bp = compiler.determinePlatform(settings, "ldmd2", "");
 		version (X86) assert(bp.architecture.canFind("x86"), bp.architecture.to!string);
 		version (X86_64) assert(bp.architecture.canFind("x86_64"), bp.architecture.to!string);
@@ -212,7 +225,7 @@ config    /etc/dmd.conf
 		}
 
 		if (!(fields & BuildSetting.cImportPaths)) {
-			settings.addDFlags(settings.cImportPaths.map!(s => "-I"~s)().array());
+			settings.addDFlags(settings.cImportPaths.map!(s => "-P-I"~s)().array());
 			settings.cImportPaths = null;
 		}
 
@@ -396,4 +409,8 @@ config    /etc/dmd.conf
 				    || arg.startsWith("-defaultlib=");
 		}
 	}
+
+	protected string[] defaultProbeArgs () const {
+		return ["-quiet", "-c", "-o-", "-v"];
+	}
 }

source/dub/compilers/gdc.d

@@ -76,13 +76,15 @@ class GDCCompiler : Compiler {
 			case "x86": arch_flags = ["-m32"]; break;
 			case "x86_64": arch_flags = ["-m64"]; break;
 		}
-		settings.addDFlags(arch_flags);
 
-		return probePlatform(
-			compiler_binary,
-			arch_flags ~ ["-fsyntax-only", "-v"],
-			arch_override
-		);
+		auto bp = probePlatform(compiler_binary, arch_flags);
+
+		bool keep_arch;
+		if (arch_flags.length)
+			keep_arch = bp.architecture != probePlatform(compiler_binary, []).architecture;
+		settings.maybeAddArchFlags(keep_arch, arch_flags, arch_override);
+
+		return bp;
 	}
 
 	void prepareBuildSettings(ref BuildSettings settings, const scope ref BuildPlatform platform, BuildSetting fields = BuildSetting.all) const
@@ -266,6 +268,10 @@ class GDCCompiler : Compiler {
 
 		return  dflags;
 	}
+
+	protected string[] defaultProbeArgs () const {
+		return ["-fsyntax-only", "-v"];
+	}
 }
 
 private string extractTarget(const string[] args) { auto i = args.countUntil("-o"); return i >= 0 ? args[i+1] : null; }

source/dub/compilers/ldc.d

@@ -79,6 +79,7 @@ config    /etc/ldc2.conf (x86_64-pc-linux-gnu)
 	BuildPlatform determinePlatform(ref BuildSettings settings, string compiler_binary, string arch_override)
 	{
 		string[] arch_flags;
+		bool arch_override_is_triple = false;
 		switch (arch_override) {
 			case "": break;
 			case "x86": arch_flags = ["-march=x86"]; break;
@@ -87,19 +88,22 @@ config    /etc/ldc2.conf (x86_64-pc-linux-gnu)
 			case "aarch64": arch_flags = ["-march=aarch64"]; break;
 			case "powerpc64": arch_flags = ["-march=powerpc64"]; break;
 			default:
-				if (arch_override.canFind('-'))
+				if (arch_override.canFind('-')) {
+					arch_override_is_triple = true;
 					arch_flags = ["-mtriple="~arch_override];
-				else
+				} else
 					throw new UnsupportedArchitectureException(arch_override);
 				break;
 		}
-		settings.addDFlags(arch_flags);
 
-		return probePlatform(
-			compiler_binary,
-			arch_flags ~ ["-c", "-o-", "-v"],
-			arch_override
-		);
+		auto bp = probePlatform(compiler_binary, arch_flags);
+
+		bool keep_arch = arch_override_is_triple;
+		if (!keep_arch && arch_flags.length)
+			keep_arch = bp.architecture != probePlatform(compiler_binary, []).architecture;
+		settings.maybeAddArchFlags(keep_arch, arch_flags, arch_override);
+
+		return bp;
 	}
 
 	void prepareBuildSettings(ref BuildSettings settings, const scope ref BuildPlatform platform, BuildSetting fields = BuildSetting.all) const
@@ -133,7 +137,7 @@ config    /etc/ldc2.conf (x86_64-pc-linux-gnu)
 		}
 
 		if (!(fields & BuildSetting.cImportPaths)) {
-			settings.addDFlags(settings.cImportPaths.map!(s => "-I"~s)().array());
+			settings.addDFlags(settings.cImportPaths.map!(s => "-P-I"~s)().array());
 			settings.cImportPaths = null;
 		}
 
@@ -335,4 +339,8 @@ config    /etc/ldc2.conf (x86_64-pc-linux-gnu)
 				    || arg.startsWith("-mtriple=");
 		}
 	}
+
+	protected string[] defaultProbeArgs () const {
+		return ["-c", "-o-", "-v"];
+	}
 }

source/dub/data/settings.d

@@ -195,7 +195,7 @@ unittest {
     import std.exception : assertThrown;
 
     const str1 = `{
-  "skipRegistry": "all"
-`;
+  "skipRegistry": "default_"
+}`;
     assertThrown!Exception(parseConfigString!Settings(str1, "/dev/null"));
 }