Make X509 serial number parsing code compatible with YARA (avast#954)

src/fileformat/file_format/pe/authenticode/x509_certificate.cpp

@@ -24,18 +24,61 @@ X509Certificate::X509Certificate(const X509* cert)
 
 std::string X509Certificate::getSerialNumber() const
 {
-	ASN1_INTEGER* serial_number_asn1 = X509_get_serialNumber(const_cast<X509*>(cert));
-	BIGNUM* bignum = ASN1_INTEGER_to_BN(serial_number_asn1, nullptr);
-
-	BIO* bio = BIO_new(BIO_s_mem());
-	BN_print(bio, bignum);
-	auto data_len = BIO_number_written(bio);
-
-	std::vector<char> result(data_len);
-	BIO_read(bio, static_cast<void*>(result.data()), data_len);
-	BIO_free_all(bio);
-	BN_free(bignum);
-	return { result.begin(), result.end() };
+    // https://github.com/VirusTotal/yara/blob/879a6576dd6e544bf9fc7711821029bf842fac54/libyara/modules/pe/pe.c#L1316
+    ASN1_INTEGER* serial_number_asn1 = X509_get_serialNumber(const_cast<X509*>(cert));
+    if (serial_number_asn1) {
+        // ASN1_INTEGER can be negative (serial->type & V_ASN1_NEG_INTEGER),
+        // in which case the serial number will be stored in 2's complement.
+        //
+        // Handle negative serial numbers, which are technically not allowed
+        // by RFC5280, but do exist. An example binary which has a negative
+        // serial number is: 4bfe05f182aa273e113db6ed7dae4bb8.
+        //
+        // Negative serial numbers are handled by calling i2d_ASN1_INTEGER()
+        // with a NULL second parameter. This will return the size of the
+        // buffer necessary to store the proper serial number.
+        //
+        // Do this even for positive serial numbers because it makes the code
+        // cleaner and easier to read.
+
+        int bytes = i2d_ASN1_INTEGER(serial_number_asn1, nullptr);
+
+        // According to X.509 specification the maximum length for the
+        // serial number is 20 octets. Add two bytes to account for
+        // DER type and length information.
+
+        if (bytes > 2 && bytes <= 22) {
+            // Now that we know the size of the serial number allocate enough
+            // space to hold it, and use i2d_ASN1_INTEGER() one last time to
+            // hold it in the allocated buffer.
+
+            std::vector<unsigned char> serial_der(bytes, 0);
+            auto tmp_pointer = serial_der.data();
+
+            bytes = i2d_ASN1_INTEGER(serial_number_asn1, &tmp_pointer);
+
+            // Also allocate space to hold the "common" string format:
+            // 00:01:02:03:04...
+            //
+            // For each byte in the serial to convert to hexlified format we
+            // need three bytes, two for the byte itself and one for colon.
+            // The last one doesn't have the colon, but the extra byte is used
+            // for the NULL terminator.
+            std::vector<char> result(bytes * 3 - 1, 0);
+            for (int j = 0; j < bytes - 2; j++)
+            {
+                // Don't put the colon on the last one.
+                // Skip over DER type, length information (first 2 bytes of serial_der)
+                if (j < bytes - 1)
+                    snprintf(result.data() + 3 * j, 4, "%02x:", serial_der[j + 2]);
+                else
+                    snprintf(result.data() + 3 * j, 3, "%02x", serial_der[j + 2]);
+            }
+            return {result.begin(), result.end()};
+        }
+    }
+    // X509_get_serialNumber returned nullptr or i2d_ASN1_INTEGER returned invalid number of bytes
+    return {};
 }
 
 std::string X509Certificate::getSignatureAlgorithm() const