Check the that the hash length is shorter than the message length. This
avoids:
./tools/../lib/rsa/rsa-verify.c:275:11: warning:
‘*db’ may be used uninitialized [-Wmaybe-uninitialized]
275 | db[0] &= 0xff >> leftmost_bits;
Signed-off-by: Heinrich Schuchardt <heinrich.schucha...@canonical.com>
---
v2:
Use -EINVAL to indicate incorrect parameters.
Add description of return value.
---
lib/rsa/rsa-verify.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c
index 1d95cfbdee..f13cd23cf2 100644
--- a/lib/rsa/rsa-verify.c
+++ b/lib/rsa/rsa-verify.c
@@ -215,6 +215,8 @@ out:
* @msg_len: Message length
* @hash: Pointer to the expected hash
* @hash_len: Length of the hash
+ *
+ * Return: 0 if padding is correct, non-zero otherwise
*/
int padding_pss_verify(struct image_sign_info *info,
const uint8_t *msg, int msg_len,
@@ -234,6 +236,9 @@ int padding_pss_verify(struct image_sign_info *info,
uint8_t leftmost_mask;
struct checksum_algo *checksum = info->checksum;
+ if (db_len <= 0)
+ return -EINVAL;
+
/* first, allocate everything */
db_mask = malloc(db_len);
db = malloc(db_len);
--
2.37.2
