liblbx: Fix framebuffer allocation for 0x0 images.

The allocation strategy for the framebuffer is the "array2" method from
c-faq: an array of row pointers into a single large array containing all
rows.  This was not being freed correctly in the case with 0 rows, since
there would be no pointer to the row data at all in order to free it.

Fix that up by simply allocating a 1x1 framebuffer when it would
otherwise be empty.  While we're at it, also add an overflow check
on the multiplication.

diff --git a/src/image.c b/src/image.c
index 98d1a31118ceb03085d14d860045ef267ed52244..f68f2da9f9ac4b1573e9d16ed53279ac9a236043 100644 (file)
--- a/src/image.c
+++ b/src/image.c
@@ -20,6 +20,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include <inttypes.h>
 #include <errno.h>
 
 #include "pack.h"
@@ -280,6 +281,15 @@ static unsigned char **allocframebuffer(size_t width, size_t height)
        unsigned char **new, *tmp;
        size_t i;
 
+       if (height > SIZE_MAX / sizeof *new) {
+               lbx_error_raise(LBX_ENOMEM);
+               return NULL;
+       }
+
+       /* Ensure that there is at least one row in the framebuffer. */
+       if (height == 0 || width == 0)
+               width = height = 1;
+
        tmp = calloc(height, width);
        if (!tmp) {
                lbx_error_raise(LBX_ENOMEM);

diff --git a/tests/empty-image.tap b/tests/empty-image.tap
index 0ef378b877293436dc48bb1d4cf9e208cf98fd85..c506eb5797feefe294d7c7cfc1a6dcf0e4efc4bf 100755 (executable)
--- a/tests/empty-image.tap
+++ b/tests/empty-image.tap
@@ -22,8 +22,7 @@
 plan_ 2
 dx_create_testdir
 
-command_ok_ "decoding image-0x0" -D TODO \
-       $LBXIMG -F pbm -dnf "$testdata/image-0x0"
+command_ok_ "decoding image-0x0" $LBXIMG -F pbm -dnf "$testdata/image-0x0"
 
 check_output() {
        diff out.000.pbm - 1>&2 <<'EOF'