[PATCH spice-common 3/3] Avoid integer overflow computing image sizes

[Frediano Ziglio <fziglio@xxxxxxxxxx>]

Use always 64, sizes can be 32x32.

Signed-off-by: Frediano Ziglio <fziglio@xxxxxxxxxx>
---
 python_modules/demarshal.py | 14 ++++++--------
 python_modules/marshal.py   |  7 +++----
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/python_modules/demarshal.py b/python_modules/demarshal.py
index 7e73985..8d3f5cb 100644
--- a/python_modules/demarshal.py
+++ b/python_modules/demarshal.py
@@ -346,13 +346,12 @@ def write_validate_array_item(writer, container, item, scope, parent_scope, star
         rows = array.size[3]
         width_v = write_read_primitive(writer, start, container, width, scope)
         rows_v = write_read_primitive(writer, start, container, rows, scope)
-        # TODO: Handle multiplication overflow
         if bpp == 8:
-            writer.assign(nelements, "%s * %s" % (width_v, rows_v))
+            writer.assign(nelements, "(uint64_t) %s * %s" % (width_v, rows_v))
         elif bpp == 1:
-            writer.assign(nelements, "((%s + 7) / 8 ) * %s" % (width_v, rows_v))
+            writer.assign(nelements, "(((uint64_t) %s + 7U) / 8U ) * %s" % (width_v, rows_v))
         else:
-            writer.assign(nelements, "((%s * %s + 7) / 8 ) * %s" % (bpp, width_v, rows_v))
+            writer.assign(nelements, "((%sU * (uint64_t) %s + 7U) / 8U ) * %s" % (bpp, width_v, rows_v))
     elif array.is_bytes_length():
         is_byte_size = True
         v = write_read_primitive(writer, start, container, array.size[1], scope)
@@ -713,13 +712,12 @@ def read_array_len(writer, prefix, array, dest, scope, is_ptr):
         rows = array.size[3]
         width_v = dest.get_ref(width)
         rows_v = dest.get_ref(rows)
-        # TODO: Handle multiplication overflow
         if bpp == 8:
-            writer.assign(nelements, "%s * %s" % (width_v, rows_v))
+            writer.assign(nelements, "((uint64_t) %s * %s)" % (width_v, rows_v))
         elif bpp == 1:
-            writer.assign(nelements, "((%s + 7) / 8 ) * %s" % (width_v, rows_v))
+            writer.assign(nelements, "(((uint64_t) %s + 7U) / 8U ) * %s" % (width_v, rows_v))
         else:
-            writer.assign(nelements, "((%s * %s + 7) / 8 ) * %s" % (bpp, width_v, rows_v))
+            writer.assign(nelements, "((%sU * (uint64_t) %s + 7U) / 8U ) * %s" % (bpp, width_v, rows_v))
     elif array.is_bytes_length():
         writer.assign(nelements, dest.get_ref(array.size[2]))
     else:
diff --git a/python_modules/marshal.py b/python_modules/marshal.py
index 402273c..fd3416a 100644
--- a/python_modules/marshal.py
+++ b/python_modules/marshal.py
@@ -172,13 +172,12 @@ def get_array_size(array, container_src):
         rows = array.size[3]
         width_v = container_src.get_ref(width)
         rows_v = container_src.get_ref(rows)
-        # TODO: Handle multiplication overflow
         if bpp == 8:
-            return "(unsigned) (%s * %s)" % (width_v, rows_v)
+            return "((uint64_t) %s * %s)" % (width_v, rows_v)
         elif bpp == 1:
-            return "(unsigned) (((%s + 7) / 8 ) * %s)" % (width_v, rows_v)
+            return "((((uint64_t) %s + 7U) / 8U ) * %s)" % (width_v, rows_v)
         else:
-            return "(unsigned) (((%s * %s + 7) / 8 ) * %s)" % (bpp, width_v, rows_v)
+            return "((((uint64_t) %s * %s + 7U) / 8U ) * %s)" % (bpp, width_v, rows_v)
     elif array.is_bytes_length():
         return container_src.get_ref(array.size[2])
     else:
-- 
2.14.3

_______________________________________________
Spice-devel mailing list
Spice-devel@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/spice-devel