Merge pull request #9195 from rlm2002/zd20508

address undefined shift behavior and overflow

Author: JacobBarthelmeh

wolfcrypt/src/pwdbased.c

@@ -816,9 +816,16 @@ int wc_scrypt(byte* output, const byte* passwd, int passLen,
         ret = MEMORY_E;
         goto end;
     }
+
+    /* Check that (1 << cost) * bSz won't overflow or exceed allowed max */
+    if (((size_t)1 << cost) * (size_t)bSz > SCRYPT_WORD32_MAX) {
+        ret = BAD_FUNC_ARG;
+        goto end;
+    }
+
     /* Temporary for scryptROMix. */
-    v = (byte*)XMALLOC((size_t)((1U << cost) * bSz), NULL,
-                       DYNAMIC_TYPE_TMP_BUFFER);
+    v = (byte*)XMALLOC(((size_t)1 << cost) * (size_t)bSz, NULL,
+                         DYNAMIC_TYPE_TMP_BUFFER);
     if (v == NULL) {
         ret = MEMORY_E;
         goto end;
@@ -841,7 +848,8 @@ int wc_scrypt(byte* output, const byte* passwd, int passLen,
 
     /* Step 2. */
     for (i = 0; i < parallel; i++)
-        scryptROMix(blocks + i * (int)bSz, v, y, (int)blockSize, 1U << cost);
+        scryptROMix(blocks + i * (int)bSz, v, y, (int)blockSize,
+                    (word32)((size_t)1 << cost));
 
     /* Step 3. */
     ret = wc_PBKDF2(output, passwd, passLen, blocks, (int)blocksSz, 1, dkLen,

wolfcrypt/test/test.c

@@ -27555,6 +27555,11 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t scrypt_test(void)
         return WC_TEST_RET_ENC_EC(ret);
     if (XMEMCMP(derived, verify4, sizeof(verify4)) != 0)
         return WC_TEST_RET_ENC_NC;
+
+    ret = wc_scrypt(derived,(byte*)"pleaseletmein", 13,
+                    (byte*)"SodiumChloride", 14, 22, 8, 1, sizeof(derived));
+    if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG))
+        return WC_TEST_RET_ENC_EC(ret);
 #endif
 #else
 #ifdef SCRYPT_TEST_ALL