Fix console launcher not consulting eclipse.ini to read vmargs when t…

…here is no console window (#117)

Processes on Windows can be created without a window using the CREATE_NO_WINDOW flag. When the
console launcher eclipsec.exe is started as such a process, the isConsoleLauncher function wrongly
returns 0 because there is no console window and hence, only eclipsec.ini is consulted for vmargs
but not eclipse.ini (see also functions getConfigArgs and getIniFile).

The fix is to properly propagate the launcher mode from main to the launcher DLL and removing the
incomplete isConsoleLauncher function entirely.

features/org.eclipse.equinox.executable.feature/library/eclipse.c

@@ -392,15 +392,15 @@ static _TCHAR** extractVMArgs(_TCHAR** launcherIniValues);
 #ifdef _WIN32
 static void     createConsole();
 static void		fixDLLSearchPath();
-static int 		isConsoleLauncher();
 #endif
-static int      consoleLauncher = 0;
+static int      isConsoleLauncher = 0;
 
 /* Record the arguments that were used to start the original executable */
-JNIEXPORT void setInitialArgs(int argc, _TCHAR** argv, _TCHAR* lib) {
+JNIEXPORT void setInitialArgs(int argc, _TCHAR** argv, _TCHAR* lib, int consoleLauncher) {
 	initialArgc = argc;
 	initialArgv = argv;
 	eclipseLibrary = lib;
+	isConsoleLauncher = consoleLauncher;
 }
 
 #ifdef MACOSX
@@ -571,9 +571,6 @@ static int _run(int argc, _TCHAR* argv[], _TCHAR* vmArgs[])
    	 */
     initWindowSystem( &argc, argv, !noSplash );
 #elif _WIN32
-    /* this must be before doing any console stuff, platforms other than win32 leave this set to 0 */
-    consoleLauncher = isConsoleLauncher();
-
     /*fix the DLL search path for security */
     fixDLLSearchPath();
 #endif
@@ -1017,7 +1014,7 @@ static _TCHAR** getConfigArgs() {
 	int configArgc = 0;
 	int ret = 0;
 
-	configFile = (iniFile != NULL) ? iniFile : getIniFile(program, consoleLauncher);
+	configFile = (iniFile != NULL) ? iniFile : getIniFile(program, isConsoleLauncher);
 	ret = readConfigFile(configFile, &configArgc, &configArgv);
 	if (ret == 0)
 		return configArgv;
@@ -1603,20 +1600,6 @@ static void createConsole() {
 	}
 }
 
-/* Determine if the launcher was the eclipsec.exe or not based on whether we have an attached console.
- * This will only be correct if called before createConsole.
- */
-static int isConsoleLauncher() {
-	HWND (WINAPI *GetConsoleWindow)();
-	void * handle = loadLibrary(_T_ECLIPSE("Kernel32.dll"));
-	if (handle != NULL) {
-		if ( (GetConsoleWindow = findSymbol(handle, _T_ECLIPSE("GetConsoleWindow"))) != NULL) {
-			return GetConsoleWindow() != NULL;
-		}
-	}
-	return 0;
-}
-
 static void fixDLLSearchPath() {
 #ifdef UNICODE
 	_TCHAR* functionName = _T_ECLIPSE("SetDllDirectoryW");
@@ -1648,7 +1631,7 @@ static int vmEEProps(_TCHAR * eeFile, _TCHAR ** msg) {
 			return LAUNCH_JNI;
 	}
 
-	if (eeConsole != NULL && (debug || needConsole || consoleLauncher) ) {
+	if (eeConsole != NULL && (debug || needConsole || isConsoleLauncher) ) {
 		javaVM = findSymlinkCommand(eeConsole, 0);
 		if (javaVM != NULL)
 			return LAUNCH_EXE;
@@ -1679,7 +1662,7 @@ static int determineVM(_TCHAR** msg) {
 	int type = 0;
 
 #ifdef _WIN32
-	if (debug || needConsole || consoleLauncher)
+	if (debug || needConsole || isConsoleLauncher)
 		defaultJava = consoleVM; /* windows will want java.exe for the console, not javaw.exe */
 #endif
 

features/org.eclipse.equinox.executable.feature/library/eclipseConfig.c

@@ -38,18 +38,14 @@
 
 #endif
 
-int readIniFile(_TCHAR* program, int *argc, _TCHAR ***argv) 
+int readIniFile(_TCHAR* program, int *argc, _TCHAR ***argv, int consoleLauncher) 
 {
 	_TCHAR* config_file = NULL;
 	int result;
 
 	if (program == NULL || argc == NULL || argv == NULL) return -1;
-
-#if defined(_WIN32) && defined(_WIN32_CONSOLE)	
-	config_file = getIniFile(program, 1);
-#else
-	config_file = getIniFile(program, 0);
-#endif
+
+	config_file = getIniFile(program, consoleLauncher);
 
 	result = readConfigFile(config_file, argc, argv);
 	free(config_file);

features/org.eclipse.equinox.executable.feature/library/eclipseConfig.h

@@ -33,7 +33,7 @@
  *
  * Returns 0 if success.
  */
-extern int readIniFile(_TCHAR* program, int *argc, _TCHAR ***argv);
+extern int readIniFile(_TCHAR* program, int *argc, _TCHAR ***argv, int consoleLauncher);
 
 /**
  * Return the path to the launcher ini file for the corresponding program 

features/org.eclipse.equinox.executable.feature/library/eclipseMain.c

@@ -56,7 +56,7 @@ administrative privileges.");
 
 /* this typedef must match the run method in eclipse.c */
 typedef int (*RunMethod)(int argc, _TCHAR* argv[], _TCHAR* vmArgs[]);
-typedef void (*SetInitialArgs)(int argc, _TCHAR*argv[], _TCHAR* library);
+typedef void (*SetInitialArgs)(int argc, _TCHAR*argv[], _TCHAR* library, int consoleLauncher);
 
 static _TCHAR*  name          = NULL;			/* program name */
 static _TCHAR** userVMarg     = NULL;     		/* user specific args for the Java VM */
@@ -120,6 +120,11 @@ int main( int argc, _TCHAR* argv[] )
 	_TCHAR*  ch;
 	_TCHAR** configArgv = NULL;
 	int 	 configArgc = 0;
+#if defined(_WIN32) && defined(_WIN32_CONSOLE)
+	int      consoleLauncher = 1;
+#else
+	int      consoleLauncher = 0;
+#endif
 	int      exitCode = 0;
 	int      ret = 0;
 	void *	 handle = 0;
@@ -152,7 +157,7 @@ int main( int argc, _TCHAR* argv[] )
     if (iniFile != NULL)
 		ret = readConfigFile(iniFile, &configArgc, &configArgv);
     else
-		ret = readIniFile(program, &configArgc, &configArgv);
+		ret = readIniFile(program, &configArgc, &configArgv, consoleLauncher);
 	if (ret == 0)
 	{
 		parseArgs (&configArgc, configArgv, 0);
@@ -206,7 +211,7 @@ int main( int argc, _TCHAR* argv[] )
 
 	setArgs = (SetInitialArgs)findSymbol(handle, SET_INITIAL_ARGS);
 	if(setArgs != NULL)
-		setArgs(initialArgc, initialArgv, eclipseLibrary);
+		setArgs(initialArgc, initialArgv, eclipseLibrary, consoleLauncher);
 	else {
 		if(!suppressErrors)
 			displayMessage(officialName, entryMsg);

features/org.eclipse.equinox.executable.feature/library/win32/make_win64.mak

@@ -38,6 +38,7 @@ PROGRAM_LIBRARY = eclipse_$(LIB_VERSION).dll
 
 # Define the object modules to be compiled and flags.
 MAIN_OBJS = eclipseMain.obj
+MAIN_CONSOLE_OBJS = eclipseMainConsole.obj
 COMMON_OBJS = eclipseConfig.obj eclipseCommon.obj   eclipseWinCommon.obj
 DLL_OBJS	= eclipse.obj  eclipseWin.obj  eclipseUtil.obj  eclipseJNI.obj eclipseShm.obj
 
@@ -67,6 +68,9 @@ all: $(EXEC) $(DLL) $(CONSOLE)
 eclipseMain.obj: ../eclipseUnicode.h ../eclipseCommon.h ../eclipseMain.c 
 	cl $(DEBUG) $(wcflags) $(cvarsmt) /Fo$*.obj ../eclipseMain.c
 
+eclipseMainConsole.obj: ../eclipseUnicode.h ../eclipseCommon.h ../eclipseMain.c 
+	cl $(DEBUG) $(wcflags) $(cvarsmt) -D_WIN32_CONSOLE /Fo$*.obj ../eclipseMain.c
+
 eclipseCommon.obj: ../eclipseCommon.h ../eclipseUnicode.h ../eclipseCommon.c
 	cl $(DEBUG) $(wcflags) $(cvarsmt) /Fo$*.obj ../eclipseCommon.c
 
@@ -94,11 +98,8 @@ eclipseShm.obj: ../eclipseShm.h ../eclipseUnicode.h ../eclipseShm.c
 $(EXEC): $(MAIN_OBJS) $(COMMON_OBJS) $(RES)
     link $(LFLAGS) -out:$(PROGRAM_OUTPUT) $(MAIN_OBJS) $(COMMON_OBJS) $(RES) $(LIBS)
 
-#the console version needs a flag set, should look for a better way to do this
-$(CONSOLE): $(MAIN_OBJS) $(COMMON_OBJS)
-	del -f eclipseConfig.obj aeclipseConfig.obj
-	cl $(DEBUG) $(wcflags) $(cvarsmt) -D_WIN32_CONSOLE /FoeclipseConfig.obj ../eclipseConfig.c
-    link $(CONSOLEFLAGS) -out:$(CONSOLE) $(MAIN_OBJS) $(COMMON_OBJS) $(LIBS)
+$(CONSOLE): $(MAIN_CONSOLE_OBJS) $(COMMON_OBJS)
+    link $(CONSOLEFLAGS) -out:$(CONSOLE) $(MAIN_CONSOLE_OBJS) $(COMMON_OBJS) $(LIBS)
 
 $(DLL): $(DLL_OBJS) $(COMMON_OBJS)
     link $(DLL_LFLAGS) -out:$(PROGRAM_LIBRARY) $(DLL_OBJS) $(COMMON_OBJS) $(DLL_LIBS)
@@ -111,4 +112,4 @@ install: all
 	del -f $(EXEC) $(MAIN_OBJS) $(DLL_OBJS) $(COMMON_OBJS) $(RES)
 
 clean:
-	del $(EXEC) $(DLL) $(MAIN_OBJS) $(DLL_OBJS) $(COMMON_OBJS) $(RES)
+	del $(EXEC) $(DLL) $(MAIN_OBJS) $(MAIN_CONSOLE_OBJS) $(DLL_OBJS) $(COMMON_OBJS) $(RES)