malloc: Use overflow checking primitives where we do complex allocations

This attempts to fix the places where we do the following where
arithmetic_expr may include unvalidated data:

  X = grub_malloc(arithmetic_expr);

It accomplishes this by doing the arithmetic ahead of time using grub_add(),
grub_sub(), grub_mul() and testing for overflow before proceeding.

Among other issues, this fixes:
  - allocation of integer overflow in grub_video_bitmap_create()
    reported by Chris Coulson,
  - allocation of integer overflow in grub_png_decode_image_header()
    reported by Chris Coulson,
  - allocation of integer overflow in grub_squash_read_symlink()
    reported by Chris Coulson,
  - allocation of integer overflow in grub_ext2_read_symlink()
    reported by Chris Coulson,
  - allocation of integer overflow in read_section_as_string()
    reported by Chris Coulson.

Fixes: CVE-2020-14309, CVE-2020-14310, CVE-2020-14311

Signed-off-by: Peter Jones <pjones@redhat.com>
Reviewed-by: Daniel Kiper <daniel.kiper@oracle.com>

grub-core/fs/btrfs.c

@@ -40,6 +40,7 @@
 #include <grub/btrfs.h>
 #include <grub/crypto.h>
 #include <grub/diskfilter.h>
+#include <grub/safemath.h>
 
 GRUB_MOD_LICENSE ("GPLv3+");
 
@@ -331,9 +332,13 @@ save_ref (struct grub_btrfs_leaf_descriptor *desc,
   if (desc->allocated < desc->depth)
     {
       void *newdata;
-      desc->allocated *= 2;
-      newdata = grub_realloc (desc->data, sizeof (desc->data[0])
-			      * desc->allocated);
+      grub_size_t sz;
+
+      if (grub_mul (desc->allocated, 2, &desc->allocated) ||
+	  grub_mul (desc->allocated, sizeof (desc->data[0]), &sz))
+	return GRUB_ERR_OUT_OF_RANGE;
+
+      newdata = grub_realloc (desc->data, sz);
       if (!newdata)
 	return grub_errno;
       desc->data = newdata;
@@ -624,16 +629,21 @@ find_device (struct grub_btrfs_data *data, grub_uint64_t id)
   if (data->n_devices_attached > data->n_devices_allocated)
     {
       void *tmp;
-      data->n_devices_allocated = 2 * data->n_devices_attached + 1;
-      data->devices_attached
-	= grub_realloc (tmp = data->devices_attached,
-			data->n_devices_allocated
-			* sizeof (data->devices_attached[0]));
+      grub_size_t sz;
+
+      if (grub_mul (data->n_devices_attached, 2, &data->n_devices_allocated) ||
+	  grub_add (data->n_devices_allocated, 1, &data->n_devices_allocated) ||
+	  grub_mul (data->n_devices_allocated, sizeof (data->devices_attached[0]), &sz))
+	goto fail;
+
+      data->devices_attached = grub_realloc (tmp = data->devices_attached, sz);
       if (!data->devices_attached)
 	{
+	  data->devices_attached = tmp;
+
+ fail:
 	  if (ctx.dev_found)
 	    grub_device_close (ctx.dev_found);
-	  data->devices_attached = tmp;
 	  return NULL;
 	}
     }