Symbol command-line tests

scripts/clear_config.cmd

@@ -0,0 +1,12 @@
+REM Clear the saved config options (in the registry)
+SET "TMPOUT=%TEMP%\set_config.out"
+
+REG DELETE "HKCU\SOFTWARE\codersnotes.com\Very Sleepy" /F >"%TMPOUT%" 2>&1
+if ERRORLEVEL 1 (
+	rem  reg.exe outputs "The operation completed successfully." on success.
+	rem    This is nice, but is undesirable for our tests.
+	rem    Only emit the output if something fails.
+	@echo %~n0: failure
+	@type "%TMPOUT%"
+	exit /b !ERRORLEVEL!
+)

scripts/clear_env.cmd

@@ -13,3 +13,9 @@ set VC=
 set VS_YEAR=
 set CC=
 set MAKE=
+
+rem Ensure these don't interfere with tests.
+rem   https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/general-environment-variables
+set _NT_SYMBOL_PATH=
+set _NT_ALT_SYMBOL_PATH=
+set _NT_SYMBOL_PROXY=

scripts/get_config.cmd

@@ -0,0 +1,13 @@
+@SETLOCAL ENABLEDELAYEDEXPANSION
+@ECHO OFF
+
+REM  Retrieve a config value from the registry and store it in the specified environment variable.
+REM  Usage: get_config.cmd <option_name> <target_envvar>
+
+SET "VALUE_NAME=%~1"
+SET "VAR_NAME=%~2"
+
+FOR /F "usebackq tokens=*" %%v IN (`PowerShell.exe -NoProfile -ExecutionPolicy Bypass -Command "Get-ItemPropertyValue -Path 'Registry::HKEY_CURRENT_USER\SOFTWARE\codersnotes.com\Very Sleepy' -Name '!VALUE_NAME!' -ErrorAction:SilentlyContinue" ^<NUL`) DO SET "VALUE=%%~v"
+
+ENDLOCAL & SET "%VAR_NAME%=%VALUE%"
+

scripts/get_stat.cmd

@@ -0,0 +1,27 @@
+@setlocal enabledelayedexpansion
+@echo off
+
+rem  usage: get_stat.cmd sleepy_file_basename stat_descr target_envvar
+rem  Get `stat_descr` from Stats.txt and save its value in `target_envvar`
+
+set "SLEEPY_BASE=%~1"
+set "STAT_DESCR=%~2"
+set "TARGET_ENVVAR=%~3"
+
+if exist "!SLEEPY_BASE!" rmdir /S /Q "!SLEEPY_BASE!"
+call %~dp0\unpack_to "!SLEEPY_BASE!.sleepy" "!SLEEPY_BASE!" > nul
+if errorlevel 1 exit 1
+
+set FOUND=0
+set STAT_VALUE=_NOT_FOUND_
+if exist "!SLEEPY_BASE!\Stats.txt" (
+	for /f "tokens=1,2* delims=: " %%a in (!SLEEPY_BASE!\Stats.txt) do (
+		 if "%%~a"=="!STAT_DESCR!" (
+			 set FOUND=1
+			 set "STAT_VALUE=%%~b"
+		 )
+	)
+)
+if %FOUND% == 0 exit /b 1
+
+ENDLOCAL & set "%TARGET_ENVVAR%=%STAT_VALUE%"

scripts/set_config.cmd

@@ -0,0 +1,19 @@
+@setlocal enabledelayedexpansion
+@echo off
+
+rem  Set a config value (that's a string/REG_SZ) in the registry.
+rem  Usage: set_config.cmd option_name new_value
+
+set "VALUE_NAME=%~1"
+set "VALUE_DATA=%~2"
+
+SET "TMPOUT=%TEMP%\set_config.out"
+reg add "HKCU\SOFTWARE\codersnotes.com\Very Sleepy" /v "!VALUE_NAME!" /t REG_SZ /d "!VALUE_DATA!" /f >"%TMPOUT%" 2>&1
+if ERRORLEVEL 1 (
+	rem  reg.exe outputs "The operation completed successfully." on success.
+	rem	 This is nice, but is undesirable for our tests.
+	rem	 Only emit the output if something fails.
+	@type "%TMPOUT%"
+	exit /b !ERRORLEVEL!
+)
+

scripts/set_config_dword.cmd

@@ -0,0 +1,19 @@
+@setlocal enabledelayedexpansion
+@echo off
+
+rem  Set a config value (that's a DWORD) in the registry.
+rem  Usage: set_config_dword.cmd <option_name> <new_value>
+
+set "VALUE_NAME=%~1"
+set "VALUE_DATA=%~2"
+
+SET "TMPOUT=%TEMP%\set_config.out"
+reg add "HKCU\SOFTWARE\codersnotes.com\Very Sleepy" /v "!VALUE_NAME!" /t REG_DWORD /d !VALUE_DATA! /f >"%TMPOUT%" 2>&1
+if ERRORLEVEL 1 (
+	rem  reg.exe outputs "The operation completed successfully." on success.
+	rem	 This is nice, but is undesirable for our tests.
+	rem	 Only emit the output if something fails.
+	@type "%TMPOUT%"
+	exit /b !ERRORLEVEL!
+)
+

scripts/test_minidump.cmd

@@ -0,0 +1,12 @@
+@echo off
+
+rem  usage: test_minidump.cmd <sleepy_file_basename>
+
+if exist %1 rmdir /S /Q %1
+call %~dp0\unpack_to %1.sleepy %1 > nul
+if errorlevel 1 exit /b 1
+
+set FOUND=0
+if exist "%~1\minidump.dmp" set FOUND=1
+
+if %FOUND% == 0 exit /b 1

scripts/test_symbol.cmd

@@ -2,7 +2,7 @@
 
 if exist %1 rmdir /S /Q %1
 call %~dp0\unpack_to %1.sleepy %1 > nul
-if errorlevel 1 exit 1
+if errorlevel 1 exit /b 1
 
 set FOUND=0
 for /F "tokens=3" %%a in (%1\Symbols.txt) do if "%%~a" == "%~2" set FOUND=1

tests/cmdline/.gitignore

@@ -0,0 +1,4 @@
+*.ilk
+
+# custom script to build (outside of appveyor environment)
+build.local.bat

tests/cmdline/ReadMe.md

@@ -0,0 +1,30 @@
+# Command Line Parameter Testing
+
+These command line parameters are tested
+
+  * minidump
+  * samplerate
+  * symopts
+
+The tests are not so much tests of functionality related to the parameters, but rather tests that the parameters are correctly honored when specified on the command line.
+
+
+
+## Overridable Options
+
+Some command line parameters will override what is saved in the application config.
+
+When you specify one of those parameters on the command line, for that instance: the application should not permit you to change the value, nor should it change the value in the saved config.
+
+These parameters exhibit this behavior
+
+  * `/minidump`
+  * `/samplerate`
+  * `/symsearchpath`
+  * `/symcachedir`
+  * `/usesymserver` (specify `/usesymserver-` to turn it off)
+  * `/symserver`
+
+You can verify the behavior by specifying one or more of the parameters on the command line, then going into Tools / Options, and verify that the corresponding setting shows the override value and cannot be changed.
+
+You can also verify that the saved config data for that setting does not get updated with the value specified on the command line.

tests/cmdline/build.bat

@@ -0,0 +1,21 @@
+@setlocal
+@echo off
+cd /d "%~dp0"
+call ..\..\scripts\clear_env.cmd
+
+rem  Build executables for two different architectures.
+rem    Ensure that the .pdb is _not_ in the same dir as the executable.
+
+SETLOCAL
+call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86
+cl /MD /Z7 program.c /Fe:program32.exe /link /DEBUG:FASTLINK
+if not exist "%~dp0symdir32" mkdir "%~dp0symdir32"
+move /y program32.pdb "%~dp0symdir32"
+ENDLOCAL
+
+SETLOCAL
+call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64
+cl /MD /Z7 program.c /Fe:program64.exe /link /DEBUG:FASTLINK
+if not exist "%~dp0symdir64" mkdir "%~dp0symdir64"
+move /y program64.pdb "%~dp0symdir64"
+ENDLOCAL

tests/cmdline/program.c

@@ -0,0 +1,13 @@
+int vs_test_fun()
+{
+	unsigned int i, n = 17;
+	// This is 4x longer than the other ones because we need more time to gather samples.
+	for (i=0; i<4000000000u; i++)
+		n = n * 17 + i;
+	return n;
+}
+
+int main()
+{
+	return vs_test_fun();
+}

tests/cmdline/symcache32/program32.pdb/9100AD1E7D7F41A595C3D1A186DA1D773/refs.ptr

@@ -0,0 +1 @@
+0000000001,file,"c:\data\src\work\VerySleepy\verysleepy-mojomojomojo\tests\tests\cmdline\symdir32\program32.pdb",pri,,N,,

tests/cmdline/symcache64/program64.pdb/EAD8BF2DA1C44033B73F323250735FE03/refs.ptr

@@ -0,0 +1 @@
+0000000001,file,"c:\data\src\work\VerySleepy\verysleepy-mojomojomojo\tests\tests\cmdline\symdir64\program64.pdb",pri,,N,,

tests/cmdline/test.bat

@@ -0,0 +1,28 @@
+@echo off
+setlocal enabledelayedexpansion
+if not exist program32.exe call build
+if not exist program64.exe call build
+
+rem https://github.com/VerySleepy/verysleepy/issues/28
+
+rem Verify that command-line options are honored and do not override saved configuration settings.
+
+set SLEEPY_SILENT_CRASH=1
+
+for %%b in (32 64) do (
+	echo.
+	echo ============================================================
+	echo Testing %%b
+	echo.
+	echo.
+
+	FOR %%o in (minidump samplerate symopts) do (
+		rem  Discard any saved config options.
+		call ..\..\scripts\clear_config
+
+		call test_cmdline_%%o.cmd %%b
+		if !ERRORLEVEL! neq 0 exit /b 1
+	)
+)
+
+echo %~f0: OK

tests/cmdline/test_cmdline_minidump.cmd

@@ -0,0 +1,66 @@
+@echo off
+setlocal enabledelayedexpansion
+
+set "PGMARCH=%~1"
+
+set "SCRIPTS_DIR=%~dps0..\..\scripts"
+
+call %SCRIPTS_DIR%\get_config SaveMinidump START_MINIDUMP_VALUE
+IF NOT "!START_MINIDUMP_VALUE!"=="" (IF "START_MINIDUMP_VALUE"=="429496729" (set /a START_MINIDUMP_VALUE=-1) ELSE (set /a START_MINIDUMP_VALUE=START_MINIDUMP_VALUE))
+
+rem
+rem  Test with the behavior turned OFF.
+rem
+
+set TEST_MINIDUMP_VALUE=2
+
+if "!START_MINIDUMP_VALUE!"=="!TEST_MINIDUMP_VALUE!" (set /a TEST_MINIDUMP_VALUE=TEST_MINIDUMP_VALUE+1)
+@echo Test MiniDump value: !TEST_MINIDUMP_VALUE!
+
+if exist program-!PGMARCH!.sleepy del program-!PGMARCH!.sleepy
+rem  Ensure that we profile a matching-architecture executable so that the "mismatch" warning doesn't halt the test.
+call %SCRIPTS_DIR%\sleepy!pgmarch! /r:program!pgmarch!.exe /o:program-!pgmarch!.sleepy /minidump:!TEST_MINIDUMP_VALUE!
+if !ERRORLEVEL! neq 0 exit /b 1
+
+rem  Verify the capture has a dump.
+call %SCRIPTS_DIR%\test_minidump program-!pgmarch!
+if !ERRORLEVEL! neq 0 ( @echo	Minidump not taken & exit /b 1 ) else ( @echo	Minidump taken )
+
+rem  Verify that the config value didn't change to what we specified.
+rem  It will change if it wasn't set, but not to what we specified.
+call %SCRIPTS_DIR%\get_config SaveMinidump FINAL_MINIDUMP_VALUE
+
+if "!FINAL_MINIDUMP_VALUE!"=="!TEST_MINIDUMP_VALUE!" (
+	@echo ERROR: SaveMinidump value was changed to the cmdline override value.
+	exit /b 1
+)
+
+
+rem
+rem  Test with the behavior turned OFF.
+rem
+
+rem	Anything less than 0 means "no minidump"
+rem	We want to use something different than what's in the saved config (often, -1).
+set TEST_MINIDUMP_VALUE=-2
+
+if "!START_MINIDUMP_VALUE!"=="!TEST_MINIDUMP_VALUE!" (set /a TEST_MINIDUMP_VALUE=TEST_MINIDUMP_VALUE+1)
+@echo Test MiniDump value: !TEST_MINIDUMP_VALUE!
+
+if exist program-!PGMARCH!.sleepy del program-!PGMARCH!.sleepy
+rem  Ensure that we profile a matching-architecture executable so that the "mismatch" warning doesn't halt the test.
+call %SCRIPTS_DIR%\sleepy!pgmarch! /r:program!pgmarch!.exe /o:program-!pgmarch!.sleepy /minidump:!TEST_MINIDUMP_VALUE!
+if !ERRORLEVEL! neq 0 exit /b 1
+
+rem  Verify the capture does not dave a dump.
+call %SCRIPTS_DIR%\test_minidump program-!pgmarch!
+if !ERRORLEVEL! neq 1 ( @echo	Minidump taken but should not have been & exit /b 1 ) else ( @echo	Minidump not taken )
+
+rem  Verify that the config value didn't change to what we specified.
+rem  It will change if it wasn't set, but not to what we specified.
+call %SCRIPTS_DIR%\get_config SaveMinidump FINAL_MINIDUMP_VALUE
+
+if "!FINAL_MINIDUMP_VALUE!"=="!TEST_MINIDUMP_VALUE!" (
+	@echo ERROR: SaveMinidump value was changed to the cmdline override value.
+	exit /b 1
+)

tests/cmdline/test_cmdline_samplerate.cmd

@@ -0,0 +1,87 @@
+@echo off
+setlocal enabledelayedexpansion
+
+rem  Testing Parameter: samplerate (speed throttle)
+rem    The test for this involves counting the number of samples taken during captures whose durations are identical.
+rem    Test with a number different sample rates, and verify that the number of samples decreases when the sample rate decreases.
+rem      Because sample rate is only a fudge-y way of reducing induced load due to sampling, choose
+rem      values for sample rate that _should_ clearly indicate that the rate was higher.
+rem      I have also observed that sample rates above 60 or so will not always effect taking more samples.
+
+
+set "PGMARCH=%~1"
+set CONFIG_PARAM=SpeedThrottle
+
+set "SCRIPTS_DIR=%~dps0..\..\scripts"
+
+set PREVIOUS_SAMPLES=
+for /L %%r in (1, 40, 100) DO (
+   @echo Testing Sample Rate: %%r
+   call :TEST_SAMPLE_RATE %%r OBSERVED_SAMPLE_COUNT
+   if ERRORLEVEL 1 exit /b 1
+   @echo   Observed samples: !OBSERVED_SAMPLE_COUNT!
+   if not "!PREVIOUS_SAMPLES!"=="" (
+       if !PREVIOUS_SAMPLES! gtr !OBSERVED_SAMPLE_COUNT! (
+           @echo ERROR: Sample count did not decrease as the sample rate increased.
+           exit /b 1
+       )
+   )
+   set /a PREVIOUS_SAMPLES = OBSERVED_SAMPLE_COUNT
+)
+
+exit /b
+
+
+
+
+:TEST_SAMPLE_RATE
+setlocal enabledelayedexpansion
+
+rem  Set the parameter to a different value than what's being tested with.
+rem  Perform the profiling capture.
+rem  Expand the archive.
+rem  Get the number of samples taken from Stats.txt.
+rem    Assign that value to the passed-in environment variable name.
+
+SET "TEST_SAMPLERATE_VALUE=%~1"
+SET "OUTPUT_VAR=%~2"
+
+
+set /a START_SAMPLERATE_VALUE=101-!TEST_SAMPLERATE_VALUE!
+call %SCRIPTS_DIR%\set_config_dword "!CONFIG_PARAM!" !START_SAMPLERATE_VALUE!
+call %SCRIPTS_DIR%\get_config "!CONFIG_PARAM!" CHECK_START_SAMPLERATE_VALUE
+if not "!CHECK_START_SAMPLERATE_VALUE!"=="!START_SAMPLERATE_VALUE!" (
+   @echo PRECONDITION ERROR: failed to set the saved config value for !CONFIG_PARAM!
+   exit /b 1
+)
+
+@echo   Test SampleRate value: !TEST_SAMPLERATE_VALUE!
+
+if exist "program-!PGMARCH!.sleepy" del "program-!PGMARCH!.sleepy"
+if exist "program-!PGMARCH!" rmdir /s /q "program-!PGMARCH!"
+
+call %SCRIPTS_DIR%\sleepy!pgmarch! /r:program!pgmarch!.exe /o:program-!pgmarch!.sleepy /samplerate:!TEST_SAMPLERATE_VALUE!
+if !ERRORLEVEL! neq 0 (
+   @echo ERROR: Very Sleepy reports failure: !ERRORLEVEL!
+   exit /b 1
+)
+
+rem  Get the number of samples.
+call %SCRIPTS_DIR%\get_stat "program-!pgmarch!" Samples OBSERVED_SAMPLES
+if !ERRORLEVEL! neq 0 exit /b 1
+if "!OBSERVED_SAMPLES!"=="" (
+   @echo Unable to find sample count in profile stats.
+   exit /b 1
+)
+
+rem  Verify that the config value didn't change to what we specified.
+rem  It will change if it wasn't set, but not to what we specified.
+call %SCRIPTS_DIR%\get_config "!CONFIG_PARAM!" FINAL_SAMPLERATE_VALUE
+
+IF "!FINAL_SAMPLERATE_VALUE!"=="!TEST_SAMPLERATE_VALUE!" (
+   @echo ERROR: !CONFIG_PARAM! value was changed to the cmdline override value.
+   exit /b 1
+)
+
+endlocal & set /a %OUTPUT_VAR% = %OBSERVED_SAMPLES%
+exit /b