PerforceTests: Don't stop test on the first error

Don't return on the first test failure in a suite, count the failures
Don't exit 1 on the first PerforceIntegrationTests() failure, add the failures

Count the number of tests passed, ignored/filtered ones
Display progresse with prefixes for the test suite and the test in it:
[5-7][2/13] Running test 'xxx.txt'
[5-7] Success: 13 of 13 tests passed.

Add extra empty lines for readability of the logs

Author: SRombautsU

Test/Perforce/PerforceTest.pm

@@ -12,10 +12,11 @@ BEGIN {
 
 sub PerforceIntegrationTests
 {
-	$dir = $_[0];
-	$p4port = $_[1];
-	$option = $_[2];
-	$filter = $_[3];
+	$suite = $_[0];
+	$dir = $_[1];
+	$p4port = $_[2];
+	$option = $_[3];
+	$filter = $_[4];
 	$mfa = index($dir, "MultiFactorAuthentication") > -1;
 	$devnull = "> /dev/null 2>&1";
 
@@ -25,11 +26,11 @@ sub PerforceIntegrationTests
 	if ($filter ne ""
 		&& index($filter, $dir) == -1)
 	{
-		print "Not Running Perforce Integration Tests in dir:'",$dir,"' p4port:'",$p4port,"' (Not included in filter).\n";
-		return 0;	
+		print "\n[$suite] '",$dir,"' Perforce Integration Tests Ignored (p4port:'",$p4port,"') (Not included in filter).\n";
+		return 0;
 	}
 
-	print "Running Perforce Integration Tests in dir:'",$dir,"' p4port:'",$p4port,"'\n";
+	print "\n\n[$suite] '",$dir,"' Perforce Integration Tests Running (p4port:'",$p4port,"')\n";
 
 	rmtree("Test/tmp");
 	mkdir "Test/tmp";
@@ -44,7 +45,7 @@ sub PerforceIntegrationTests
 	$ENV{'P4CHARSET'} = 'utf8';
 	$ENV{'VCS_P4PASSWD'} = 'Secret';
 	$ENV{'P4EXECABS'} = getcwd() . "/" . $ENV{'P4EXEC'};
-	
+
 	if ($ENV{'TARGET'} eq "win32")
 	{
 		$ENV{'VCS_P4ROOT'} =~ s/\//\\/g;
@@ -73,7 +74,7 @@ sub PerforceIntegrationTests
 	# print "Press ENTER to continue...";
 	# <STDIN>;
 
-	$exitCode = RunTests($dir, $option, $filter);
+	$exitCode = RunTests($suite, $dir, $option, $filter);
 
 	# print "Press ENTER to continue...";
 	# <STDIN>;
@@ -85,14 +86,18 @@ sub PerforceIntegrationTests
 
 sub RunTests()
 {
-	$dir = $_[0];
-	$option = $_[1];
-	$filter = $_[2];
+	$suite = $_[0];
+	$dir = $_[1];
+	$option = $_[2];
+	$filter = $_[3];
 
 	@files = <Test/$dir/*.txt>;
 
-	$total = 0;
+	$total = @files;
+	$count = 0;
 	$success = 0;
+	$failed = 0;
+	$ignored = 0;
 
 	$pluginexec = abs_path($ENV{'P4PLUGIN'});
 	$testserver = abs_path($ENV{'TESTSERVER'});
@@ -106,13 +111,15 @@ sub RunTests()
 
 	$cwd = getcwd();
 	foreach $i (@files) {
-		if ($filter ne "" 
+		$count++;
+		if ($filter ne ""
 			&& index($i, $filter) == -1)
 		{
-			print "Not running Perforce Integration Tests in file:'",$i,"' p4port:'",$p4port,"'\n";
+			$ignored++;
+			print "\n[$suite][$count/$total] '",$i,"' Ignored (p4port:'",$p4port,"')\n";
 			next;
 		}
-		print "Running Perforce Integration Tests in file:'",$i,"' p4port:'",$p4port,"'\n";
+		print "\n[$suite][$count/$total] '",$i,"' Running (p4port:'",$p4port,"')\n";
 		chdir $cwd;
 		rmtree( $clientroot, {keep_root => 1} );
 		print "Changing working directory to: '", $clientroot,"'\n";
@@ -129,21 +136,29 @@ sub RunTests()
 		}
 		elsif ($? == -1)
 		{
-			print "Error running test '$i' : $!\n";
+			print "\n[$suite][$count/$total] '",$i,"' Error running test: $!\n\n";
 			chdir $cwd;
 			return 1;
 		}
 		else
 		{
-			print "Test failed -> stopping all tests\n";
-			chdir $cwd;
-			return 1;
+			print "\n[$suite][$count/$total] '",$i,"' Failed (p4port:'",$p4port,"')\n\n";
+			$failed++;
 		}
-		$total++;
 		RemoveExclusiveFile();
 	}
-	print "Done: $success of $total tests passed.\n";
 	chdir $cwd;
+	if ($ignored > 0)
+	{
+		print "[$suite] $ignored of $total tests ignored.\n";
+	}
+	if ($failed > 0)
+	{
+		print "[$suite] Failed: $failed of $total tests failed.\n";
+		print "[$suite]         $success of $total tests passed.\n";
+		return $failed;
+	}
+	print "[$suite] Success: $success of $total tests passed.\n";
 	return 0;
 }
 
@@ -160,7 +175,7 @@ sub AddExclusiveFile
 	open(FH, '>', 'Assets/exclusivefile.txt') or die $!;
 	print(FH 'File with exclusive open file type modifier.');
 	close(FH) or die $1;
-	
+
 	RunCommand('add -t text+l Assets/exclusivefile.txt');
 	RunCommand('submit -d "Add Assets/exclusivefile.txt." Assets/exclusivefile.txt');
 }
@@ -173,7 +188,7 @@ sub RemoveExclusiveFile
 
 sub SetupServer
 {
-	$root = $ENV{'VCS_P4ROOT'}; 
+	$root = $ENV{'VCS_P4ROOT'};
 	my $p4port = $ENV{'VCS_P4PORT'};
 	print "Setting server in '$root' listening on port '$p4port'\n";
 	rmtree($root);
@@ -190,6 +205,7 @@ sub SetupServer
 	}
 
 	my $p4d = $ENV{'P4DEXEC'};
+	print "Starting server '$p4d'\n";
 	system("$p4d -xi -r \"$root\"");
 	my $pidfile = getcwd() . "/server.pid";
 	my $pid = SpawnSubProcess($p4d, " -r \"$root\" -p $p4port --pid-file=$pidfile");
@@ -288,9 +304,9 @@ Update: 2022/07/20 11:40:48
 
 Access: 2022/07/20 11:40:48
 
-FullName:       Multi Factor Authentication
+FullName:	   Multi Factor Authentication
 
-Password:       ******
+Password:	   ******
 
 AuthMethod:		perforce+2fa
 EOF
@@ -313,9 +329,9 @@ sub SetupMFATriggers
 	$TRIGGERS =<<EOF;
 
 Triggers:
-    test-pre-2fa auth-pre-2fa auth "$mfa_script -t pre-2fa -e %quote%%email%%quote% -u %user% -h %host%"
-    test-init-2fa auth-init-2fa auth "$mfa_script -t init-2fa -e %quote%%email%%quote% -u %user% -h %host% -m %method%"
-    test-check-2fa auth-check-2fa auth "$mfa_script -t check-2fa -e %quote%%email%%quote% -u %user% -h %host% -s %scheme% -k %token%"
+	test-pre-2fa auth-pre-2fa auth "$mfa_script -t pre-2fa -e %quote%%email%%quote% -u %user% -h %host%"
+	test-init-2fa auth-init-2fa auth "$mfa_script -t init-2fa -e %quote%%email%%quote% -u %user% -h %host% -m %method%"
+	test-check-2fa auth-check-2fa auth "$mfa_script -t check-2fa -e %quote%%email%%quote% -u %user% -h %host% -s %scheme% -k %token%"
 EOF
 
 	open(FD, "| $ENV{'P4EXEC'} -p $ENV{'VCS_P4PORT'} -u mfa_test_user -P Mfau1111 triggers -i ");
@@ -341,7 +357,7 @@ Update:2013/02/19 09:13:18
 Access:2013/06/24 12:38:18
 
 Description:
-    Created by $ENV{'VCS_P4USER'}.
+	Created by $ENV{'VCS_P4USER'}.
 
 Root:$root
 
@@ -352,8 +368,8 @@ SubmitOptions:submitunchanged
 LineEnd:local
 
 View:
-    //depot/... //$ENV{'VCS_P4CLIENT'}/...
-    -//depot/Assets/excludedfile.txt  //$ENV{'VCS_P4CLIENT'}/Assets/excludedfile.txt
+	//depot/... //$ENV{'VCS_P4CLIENT'}/...
+	-//depot/Assets/excludedfile.txt  //$ENV{'VCS_P4CLIENT'}/Assets/excludedfile.txt
 EOF
 
 	open(FD, "| $ENV{'P4EXEC'} -p $ENV{'VCS_P4PORT'} -u vcs_test_user client -i ");

Test/VCSTest.pm

@@ -3,11 +3,8 @@ use PerforceTest;
 
 sub IntegrationTest
 {
-	$exitCode = PerforceIntegrationTests(@_);
-	if ($exitCode != 0)
-	{
-		exit $exitCode;
-	}
+	$failed = PerforceIntegrationTests(@_);
+	return $failed;
 }
 
 1;

build.pl

@@ -25,30 +25,30 @@
 	rmtree("Release");
 	rmtree("Build");
 	find(\&wanted, "./");
-	sub wanted 
+	sub wanted
 	{
-    	my($filename, $dirs, $suffix) = fileparse($File::Find::name, qr/\.[^.]*/);
+		my($filename, $dirs, $suffix) = fileparse($File::Find::name, qr/\.[^.]*/);
 		if (($suffix eq ".o") or ($suffix eq ".obj"))
 		{
-    		print "delete $File::Find::name","\n";
-    		unlink($_);
+			print "delete $File::Find::name","\n";
+			unlink($_);
 		}
-    }
-    unlink("PerforcePlugin");
+	}
+	unlink("PerforcePlugin");
 	exit 0;
 }
 
 if (not $target)
 {
-	if ($^O eq "darwin") 
+	if ($^O eq "darwin")
 	{
-		$target = "mac";		
+		$target = "mac";
 	}
-	elsif ($^O eq "MSWin32") 
+	elsif ($^O eq "MSWin32")
 	{
 		$target = "win32";
 	}
-	elsif ($^O eq "linux") 
+	elsif ($^O eq "linux")
 	{
 		$target = "linux64";
 	}
@@ -60,7 +60,7 @@
 {
 	unless ($test)
 	{
-		BuildMac();	
+		BuildMac();
 	}
 	else
 	{
@@ -71,7 +71,7 @@
 {
 	unless ($test)
 	{
-		BuildWin32();	
+		BuildWin32();
 	}
 	else
 	{
@@ -100,26 +100,37 @@
 		TestLinux ($target);
 	}
 }
-else 
+else
 {
-    die ("Unknown platform");
+	die ("Unknown platform");
 }
 
 sub TestPerforce()
 {
-	IntegrationTest("Plugin", "localhost:1667", $testoption, $filter);
-	IntegrationTest("Plugin", "ssl:localhost:1667", $testoption, $filter);
-	IntegrationTest("Perforce/Common", "localhost:1667", $testoption, $filter);
-	IntegrationTest("Perforce/Common", "ssl:localhost:1667", $testoption, $filter);
-	IntegrationTest("Perforce/BaseIPv4", "tcp4:localhost:1667", $testoption, $filter);
-	IntegrationTest("Perforce/SecureBaseIPv4", "ssl4:localhost:1667", $testoption, $filter);
-	IntegrationTest("Perforce/SquareBracketIPv4", "tcp4:[localhost]:1667", $testoption, $filter);
+	my $failed = 0;
+	$failed += IntegrationTest("1/8", "Plugin", "localhost:1667", $testoption, $filter);
+	$failed += IntegrationTest("2/8", "Plugin", "ssl:localhost:1667", $testoption, $filter);
+	$failed += IntegrationTest("3/8", "Perforce/Common", "localhost:1667", $testoption, $filter);
+	$failed += IntegrationTest("4/8", "Perforce/Common", "ssl:localhost:1667", $testoption, $filter);
+	$failed += IntegrationTest("5/8", "Perforce/BaseIPv4", "tcp4:localhost:1667", $testoption, $filter);
+	$failed += IntegrationTest("6/8", "Perforce/SecureBaseIPv4", "ssl4:localhost:1667", $testoption, $filter);
+	$failed += IntegrationTest("7/8", "Perforce/SquareBracketIPv4", "tcp4:[localhost]:1667", $testoption, $filter);
+	$failed += IntegrationTest("8/8", "Perforce/MultiFactorAuthentication", "localhost:1667", $testoption, $filter);
 	# Only works if DNS routes via IPv6
-	# IntegrationTest("Perforce/BaseIPv6", "tcp6:[localhost]:1667", $testoption, $filter);
+	# $failed += IntegrationTest("Perforce/BaseIPv6", "tcp6:[localhost]:1667", $testoption, $filter);
 	# Does not work in new version of Perforce server
-	# IntegrationTest("Perforce/SquareBracketIPv6", "tcp6:[::1]:1667", $testoption, $filter);
-	# IntegrationTest("Perforce/SecureSquareBracketIPv6", "ssl6:[::1]:1667", $testoption, $filter);
-	IntegrationTest("Perforce/MultiFactorAuthentication", "localhost:1667", $testoption, $filter);
+	# $failed += IntegrationTest("Perforce/SquareBracketIPv6", "tcp6:[::1]:1667", $testoption, $filter);
+	# $failed += IntegrationTest("Perforce/SecureSquareBracketIPv6", "ssl6:[::1]:1667", $testoption, $filter);
+
+	if ($failed > 0)
+	{
+		print "\nFAILURE $failed Perforce Integrations Test(s) failed!\n\n";
+		exit 1;
+	}
+	else
+	{
+		print "\nSUCCESS: All Perforce Integrations Tests passed\n\n";
+	}
 }
 
 sub BuildMac
@@ -143,9 +154,9 @@ sub TestMac
 
 sub BuildWin32
 {
-  rmtree("Build");
-  system("msbuilder.cmd", "VersionControl.sln", "P4Plugin", "Win32") && die ("Failed to build PerforcePlugin.exe");
-  system("msbuilder.cmd", "VersionControl.sln", "TestServer", "Win32") && die ("Failed to build TestServer.exe");
+	rmtree("Build");
+	system("msbuilder.cmd", "VersionControl.sln", "P4Plugin", "Win32") && die ("Failed to build PerforcePlugin.exe");
+	system("msbuilder.cmd", "VersionControl.sln", "TestServer", "Win32") && die ("Failed to build TestServer.exe");
 }
 
 sub TestWin32
@@ -194,4 +205,4 @@ ($)
 	chmod 0755, glob("Build/linux64/*");
 
 	TestPerforce();
-}
+}