Dynamic format bug fixes post Jumbo-6, from bleeding

doc/DYNAMIC

@@ -802,12 +802,34 @@ examples (this whole list is also found in dynamic.conf).
 #
 ####################################################################
 #
+#   SaltLenX86=# line   (Optional, but required IF the format needs it)
+#
+#################
+#
+#  This is almost never needed.  But if needed to have a different
+#  salt size on non-MMX/SSE builds, then this is how to do it.
+#
+####################################################################
+#
 #   MaxInputLen=# line   (Optional, but required IF the format needs it)
 #
 #################
 #
 #  If a format has a 'max' length of password (such as Cisco PIX has
-#  a max length of 16 byte PW), then this must be set.
+#  a max length of 16 byte PW), then this must be set.  If this is
+#  not set, then the format will have a max password allowed to be
+#  55-abs(saltlen)  (80-abs(saltlen) if not an SSE/MMX build)
+#
+####################################################################
+#
+#   MaxInputLenX86=# line   (Optional, but required IF the format needs it)
+#
+#################
+#
+#  Similar to the SaltLenX86, in that it allows a different password
+#  length to be force set in non-SSE builds.  So if a format should
+#  ONLY have 16 byte passwords, and no salt, then you would need
+#  to set MaxInputLen=16  and have a MaxInputLenX86=16 set also.
 #
 ####################################################################
 #

doc/OPTIONS

@@ -320,7 +320,11 @@ The different variants of list externals may be worth mentioning:
 --list=format-all-details list all the suported formats and their attributes
                           in the same sequence as with --list=format-details,
                           but in a more descriptive form readable by humans
-
+--list=format-methods[:WHICH]
+                          list all methods which are NOT default for each
+                          format.  If 'WHICH' is set, then this listing
+                          will ONLY be formats which have the 'WHICH' format
+                          overridden.
 
 --regen-lost-salts=N	Try to find password AND salt in a set of raw hashes.
 

run/dynamic.conf

@@ -60,6 +60,7 @@ Expression=md5(md5($p).md5($p))
 # Thus, we need a non-sse2 safe flag.
 Flag=MGF_NOTSSE2Safe
 Flag=MGF_KEYS_INPUT
+Flag=MGF_FULL_CLEAN_REQUIRED
 # here is the optimized 'script' to perform hash 'like' IPB but salt replaced with password.
 Func=DynamicFunc__crypt_md5
 Func=DynamicFunc__clean_input2_kwik
@@ -176,24 +177,16 @@ Test=$dynamic_1006$e9d38522b5eeec753332e576e2e0fe5d:test3
 Expression=md5(md5($p).$s) (vBulletin)
 # Flag needed here, is Salt.  There is no 'fixed' saltlen.
 Flag=MGF_SALTED
-Flag=MGF_ColonNOTValid
 Flag=MGF_KEYS_BASE16_IN1
 # vBulletin has a 'fixed' 3 byte salt, so list the fixed size
 SaltLen=3
-ColonChar=;
 # here is the optimized 'script' to perform vBulletin hash
 Func=DynamicFunc__set_input_len_32
 Func=DynamicFunc__append_salt
 Func=DynamicFunc__crypt_md5
-# NOTE, vBulletin is setup to 'override' the ':'.  USUALLY, it is good to use something
-# outside of the ASCII values from 0x20 to 0x7F.  0x1F is a 'good' choice, but it will
-# cause john.conf to have a 'non-normal' ASCII char.  Thus for this 'simple' example, I
-# have used the ';' character.  NOTE this would have the same 'problems' as the ':' character
-# if used for real, since ; is also a valid character within the salt of vBulletin.
-# NOTE to run, you MUST use the command line switch:   -field-separator-char=;
-Test=$dynamic_1007$daa61d77e218e42060c2fa198ac1feaf$SXB;test1
-Test=$dynamic_1007$de56b00bb15d6db79204bd44383469bc$T &;thatsworking
-Test=$dynamic_1007$fb685c6f469f6e549c85e4c1fb5a65a6$\\H:;test3
+Test=$dynamic_1007$daa61d77e218e42060c2fa198ac1feaf$SXB:test1
+Test=$dynamic_1007$de56b00bb15d6db79204bd44383469bc$T &:thatsworking
+Test=$dynamic_1007$fb685c6f469f6e549c85e4c1fb5a65a6$HEX$5C483A:test3
 
 ####################################################################
 # Dynamic type for algorithm used in RADIUS User-Password attrinute md5($p.$s)
@@ -221,6 +214,7 @@ Test=$dynamic_1008$658bbf9f04538d6bede09a4a52a77504$HEX$626c61680036373839303132
 [List.Generic:dynamic_1009]
 Expression=md5($s.$p) (RADIUS Responses)
 Flag=MGF_SALTED
+Saltlen=-16
 Func=DynamicFunc__clean_input
 Func=DynamicFunc__append_salt
 Func=DynamicFunc__append_keys

src/common.c

@@ -14,6 +14,8 @@ char atoi64[0x100];
 
 char itoa16[16] =
 	"0123456789abcdef";
+char itoa16u[16] =
+	"0123456789ABCDEF";
 char atoi16[0x100];
 
 static int initialized = 0;

src/common.h

@@ -53,6 +53,7 @@ typedef unsigned int ARCH_WORD_32;
  */
 extern char itoa64[64], atoi64[0x100];
 extern char itoa16[16], atoi16[0x100];
+extern char itoa16u[16]; // uppercase
 
 /*
  * Initializes the tables.

src/dynamic.h

@@ -65,6 +65,7 @@ typedef struct DYNAMIC_Constants_t
 #define MGF_UTF8                     0x08000000
 #define MGF_PASSWORD_UPCASE          0x10000000
 #define MGF_PASSWORD_LOCASE          0x20000000
+#define MGF_FULL_CLEAN_REQUIRED      0x40000000
 
 // These are special loader flags.  They specify that keys loads are 'special', and
 // do MORE than simply load keys into the keys[] array.  They may preload the keys
@@ -97,12 +98,15 @@ typedef struct DYNAMIC_Setup_t
 	DYNAMIC_Constants *pConstants;
 	unsigned flags;
 	unsigned startFlags;
-	int SaltLen;
-	int MaxInputLen;
+	int SaltLen;			// these are SSE lengths
+	int MaxInputLen;		// SSE length.  If 0, then set to 55-abs(SaltLen)
+	int MaxInputLenX86;		// if zero, then use PW len set to 80-abs(SaltLen) (or 80-abs(SaltLenX86), if it is not 0)
+	int SaltLenX86;			// if zero, then use salt len of SSE
 } DYNAMIC_Setup;
 
 int dynamic_SETUP(DYNAMIC_Setup *, struct fmt_main *pFmt);
 int dynamic_IS_VALID(int i);
+int dynamic_real_salt_length(struct fmt_main *pFmt);
 void dynamic_RESET(struct fmt_main *);
 void dynamic_DISPLAY_ALL_FORMATS();
 

src/dynamic_fmt.c

@@ -1167,10 +1167,10 @@ static void set_key(char *key, int index)
 		if (len > 55) // we never do UTF-8 -> UTF-16 in this mode
 			len = 55;
 
-		if(index==0) {
+//		if(index==0) {
 			// we 'have' to use full clean here. NOTE 100% sure why, but 10 formats fail if we do not.
-			DynamicFunc__clean_input_full();
-		}
+//			DynamicFunc__clean_input_full();
+//		}
 #if MD5_X2
 		if (index & 1)
 			strnzcpy(input_buf_X86[index>>MD5_X2].x2.b2, key, len+1);
@@ -1184,15 +1184,31 @@ static void set_key(char *key, int index)
 		len = strlen(key);
 		if (len > 55 && !(fmt_Dynamic.params.flags & FMT_UNICODE))
 			len = 55;
-		if(index==0) {
-			DynamicFunc__clean_input();
-		}
+//		if(index==0) {
+//			DynamicFunc__clean_input();
+//		}
 		keys_dirty = 1;
 		strnzcpy(((char*)(saved_key[index])), key, len+1);
 		saved_key_len[index] = len;
 	}
 }
 
+void clear_keys(void) {
+#ifdef MMX_COEF
+	if (curdat.store_keys_in_input) {
+		if (curdat.store_keys_in_input) {
+			if (dynamic_use_sse==1 || dynamic_use_sse==1)
+				return;
+		}
+	}
+	if (curdat.pSetup->flags & MGF_FULL_CLEAN_REQUIRED)
+		DynamicFunc__clean_input_full();
+	else
+		DynamicFunc__clean_input_kwik();
+#else
+	DynamicFunc__clean_input_full();
+#endif
+}
 
 /*********************************************************************************
  * Returns the key.  NOTE how it gets it depends upon if we are storing
@@ -2284,7 +2300,7 @@ struct fmt_main fmt_Dynamic =
 		set_salt,
 		set_key,
 		get_key,
-		fmt_default_clear_keys,
+		clear_keys,
 		crypt_all,
 		{
 			get_hash_0,
@@ -7109,14 +7125,20 @@ int dynamic_SETUP(DYNAMIC_Setup *Setup, struct fmt_main *pFmt)
 		}
 	}
 
-#ifdef MMX_COEF
-		pFmt->params.plaintext_length = PLAINTEXT_LENGTH;
-#else
-		pFmt->params.plaintext_length = PLAINTEXT_LENGTH_X86;
-#endif
-
 	if (Setup->MaxInputLen)
 		pFmt->params.plaintext_length = Setup->MaxInputLen;
+	else
+		pFmt->params.plaintext_length = 55 - abs(Setup->SaltLen);
+#ifndef MMX_COEF
+	if (Setup->MaxInputLenX86) {
+		pFmt->params.plaintext_length = Setup->MaxInputLenX86;
+	} else {
+		if (Setup->SaltLenX86)
+			pFmt->params.plaintext_length = 80 - abs(Setup->SaltLenX86);
+		else
+			pFmt->params.plaintext_length = 80 - abs(Setup->SaltLen);
+	}
+#endif
 
 	curdat.store_keys_in_input = !!(Setup->startFlags&MGF_KEYS_INPUT );
 	curdat.input2_set_len32 = !!(Setup->startFlags&MGF_SET_INP2LEN32);
@@ -7442,6 +7464,9 @@ static int LoadOneFormat(int idx, struct fmt_main *pFmt)
 	memcpy(pFmt, &fmt_Dynamic, sizeof(struct fmt_main));
 	dynamic_RESET(pFmt);
 
+	// Ok we need to list this as a dynamic format (even for the 'thin' formats)
+	pFmt->params.flags |= FMT_DYNAMIC;
+
 	if (idx < 1000) {
 		if (dynamic_RESERVED_PRELOAD_SETUP(idx, pFmt) != 1)
 			return 0;
@@ -7690,3 +7715,14 @@ static char *HandleCase(char *cp, int caseType)
 	}
 	return (char*)dest;
 }
+
+int dynamic_real_salt_length(struct fmt_main *pFmt) {
+	if (pFmt->params.flags & FMT_DYNAMIC) {
+		private_subformat_data *pPriv = pFmt->private.data;
+		if (pPriv == NULL || pPriv->pSetup == NULL)
+			return -1;  // not a dynamic format, or called before we have loaded them!!
+		return abs(pPriv->pSetup->SaltLen);
+	}
+	// NOT a dynamic format
+	return -1;
+}

src/dynamic_parser.c

@@ -340,6 +340,7 @@ static MD5Gen_Str_Flag_t MD5Gen_Str_Flag[] =  {
 	{ "MGF_UTF8",                         MGF_UTF8 },
 	{ "MGF_PASSWORD_UPCASE",              MGF_PASSWORD_UPCASE },
 	{ "MGF_PASSWORD_LOCASE",              MGF_PASSWORD_LOCASE },
+	{ "MGF_FULL_CLEAN_REQUIRED",          MGF_FULL_CLEAN_REQUIRED },
 	{ NULL, 0 }};
 
 static MD5Gen_Str_Flag_t MD5Gen_Str_sFlag[] =  {
@@ -518,12 +519,24 @@ int dynamic_LOAD_PARSER_FUNCTIONS_LoadLINE(struct cfg_line *_line)
 			return 1;
 		return !fprintf(stderr, "Error, Invalid SaltLen= line:  %s  \n", Line);
 	}
+	if (c == 's' && !strncasecmp(Line, "SaltLenX86=", 11))
+	{
+		if (sscanf(&Line[10], "=%d", &pSetup->SaltLenX86) == 1)
+			return 1;
+		return !fprintf(stderr, "Error, Invalid SaltLenX86= line:  %s  \n", Line);
+	}
 	if (c == 'm' && !strncasecmp(Line, "MaxInputLen=", 12))
 	{
 		if (sscanf(&Line[11], "=%d", &pSetup->MaxInputLen) == 1)
 			return 1;
 		return !fprintf(stderr, "Error, Invalid MaxInputLen= line:  %s  \n", Line);
 	}
+	if (c == 'm' && !strncasecmp(Line, "MaxInputLenX86=", 15))
+	{
+		if (sscanf(&Line[14], "=%d", &pSetup->MaxInputLenX86) == 1)
+			return 1;
+		return !fprintf(stderr, "Error, Invalid MaxInputLenX86= line:  %s  \n", Line);
+	}
 	if (c == 'e' && !strncasecmp(Line, "Expression=", 11))
 	{
 		char tmp[256];
@@ -622,7 +635,7 @@ int dynamic_LOAD_PARSER_FUNCTIONS(int which, struct fmt_main *pFmt)
 	nPreloadCnt = 0;
 	nFuncCnt = 0;
 
-	pSetup = mem_alloc_tiny(sizeof(DYNAMIC_Setup), MEM_ALIGN_NONE);
+	pSetup = mem_calloc_tiny(sizeof(DYNAMIC_Setup), MEM_ALIGN_NONE);
 
 	if (!dynamic_LOAD_PARSER_SIGNATURE(which))
 	{