Improve error handling in soter_rand() (#485)

First of all, RAND_bytes() may return -1 if the backend does not support
cryptographically strong pseudo-random generators. Currently we treat
this as success which is not okay. Report the error properly.

There's also a minor API mismatch between OpenSSL and BoringSSL. OpenSSL
still mostly uses `int` for buffer lengths while BoringSSL has updated
its API to use `size_t`. We need to check for possible overflow when
using OpenSSL, and we do not need to cast anything with BoringSSL.

Finally, while we're here, improve the documentation to clarify when
SOTER_FAIL may be returned and what it means. Other error codes are
more or less self-explanatory so we don't mention them.

src/soter/boringssl/soter_rand.c

@@ -20,14 +20,18 @@
 
 soter_status_t soter_rand(uint8_t* buffer, size_t length)
 {
-    if ((!buffer) || (!length)) {
+    int result;
+
+    if (!buffer || !length) {
         return SOTER_INVALID_PARAMETER;
     }
 
-    if (RAND_bytes(buffer, (int)length)) {
-        return SOTER_SUCCESS;
+    /* BoringSSL's RAND_bytes() accepts size_t, no need to cast */
+    result = RAND_bytes(buffer, length);
+
+    if (result < 0) {
+        return SOTER_NOT_SUPPORTED;
     }
 
-    /* For some reason OpenSSL generator failed */
-    return SOTER_FAIL;
+    return (result == 1) ? SOTER_SUCCESS : SOTER_FAIL;
 }

src/soter/openssl/soter_rand.c

@@ -16,18 +16,23 @@
 
 #include "soter/soter_rand.h"
 
+#include <limits.h>
+
 #include <openssl/rand.h>
 
 soter_status_t soter_rand(uint8_t* buffer, size_t length)
 {
-    if ((!buffer) || (!length)) {
+    int result;
+
+    if (!buffer || !length || length > INT_MAX) {
         return SOTER_INVALID_PARAMETER;
     }
 
-    if (RAND_bytes(buffer, (int)length)) {
-        return SOTER_SUCCESS;
+    result = RAND_bytes(buffer, (int)length);
+
+    if (result < 0) {
+        return SOTER_NOT_SUPPORTED;
     }
 
-    /* For some reason OpenSSL generator failed */
-    return SOTER_FAIL;
+    return (result == 1) ? SOTER_SUCCESS : SOTER_FAIL;
 }

src/soter/soter_rand.h

@@ -28,22 +28,26 @@
 /**
  * @addtogroup SOTER
  * @{
- * @defgroup SOTER_RAND random bits generator
- * @brief Routine for rundom bits generation
+ * @defgroup SOTER_RAND generating random data
+ * @brief Routines for generating random data
  * @{
  */
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
- * @brief Generates random bits
+ * @brief Generates pseudo-random bytes
  *
- * @param [out] buffer pointer to a buffer for random bits
- * @param [in] length length of the buffer
+ * @param [out] buffer pointer to the output buffer for random data
+ * @param [in]  length length of the buffer
  * @return success code
  *
- * This function generates random bits and puts them in memory pointed by buffer.
+ * This function generates cryptographically strong pseudo-random bytes
+ * and fills the provided buffer with them.
+ *
+ * SOTER_FAIL indicates that there is not enough entropy available
+ * to fill the entire buffer. Please try again later.
  */
 SOTER_API
 soter_status_t soter_rand(uint8_t* buffer, size_t length);