From 6bb309b985fcac9c28dbf50dfe70e267abf4699f Mon Sep 17 00:00:00 2001
From: "vladimir%pobox.com"
 <vladimir%pobox.com@18797224-902f-48f8-a5cc-f745e15eee43>
Date: Mon, 11 Sep 2006 21:18:31 +0000
Subject: [PATCH] b=351296,351295,348351 int overflow in canvas
 GetImageData/PutImageData, r=biesi

git-svn-id: svn://10.0.0.236/trunk@209777 18797224-902f-48f8-a5cc-f745e15eee43
---
 .../canvas/src/nsCanvasRenderingContext2D.cpp | 37 +++++++++++++------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/mozilla/content/canvas/src/nsCanvasRenderingContext2D.cpp b/mozilla/content/canvas/src/nsCanvasRenderingContext2D.cpp
index 9deda22c24d..a3263ee50f7 100644
--- a/mozilla/content/canvas/src/nsCanvasRenderingContext2D.cpp
+++ b/mozilla/content/canvas/src/nsCanvasRenderingContext2D.cpp
@@ -174,6 +174,9 @@ static NS_DEFINE_IID(kBlenderCID, NS_BLENDER_CID);
 #define STYLE_STACK_DEPTH 50
 #define STYLE_CURRENT_STACK ((mSaveCount<STYLE_STACK_DEPTH)?mSaveCount:STYLE_STACK_DEPTH-1)
 
+static PRBool CheckSaneImageSize (PRInt32 width, PRInt32 height);
+static PRBool CheckSaneSubrectSize (PRInt32 x, PRInt32 y, PRInt32 w, PRInt32 h, PRInt32 realWidth, PRInt32 realHeight);
+
 /**
  ** nsCanvasGradient
  **/
@@ -730,6 +733,10 @@ nsCanvasRenderingContext2D::SetDimensions(PRInt32 width, PRInt32 height)
 {
     Destroy();
 
+    // Check that the dimensions are sane
+    if (!CheckSaneImageSize(width, height))
+        return NS_ERROR_FAILURE;
+
     mWidth = width;
     mHeight = height;
 
@@ -2331,7 +2338,7 @@ nsCanvasRenderingContext2D::CairoSurfaceFromElement(nsIDOMElement *imgElt,
     return NS_OK;
 }
 
-static PRBool
+PRBool
 CheckSaneImageSize (PRInt32 width, PRInt32 height)
 {
     if (width <= 0 || height <= 0)
@@ -2354,6 +2361,22 @@ CheckSaneImageSize (PRInt32 width, PRInt32 height)
     return PR_TRUE;
 }
 
+/* Check that the rect [x,y,w,h] is a valid subrect of [0,0,realWidth,realHeight]
+ * without overflowing any integers and the like.
+ */
+PRBool
+CheckSaneSubrectSize (PRInt32 x, PRInt32 y, PRInt32 w, PRInt32 h, PRInt32 realWidth, PRInt32 realHeight)
+{
+    if (w <= 0 || h <= 0 || x < 0 || y < 0)
+        return PR_FALSE;
+
+    if (x >= realWidth  || w > (realWidth - x) ||
+        y >= realHeight || h > (realHeight - y))
+        return PR_FALSE;
+
+    return PR_TRUE;
+}
+
 NS_IMETHODIMP
 nsCanvasRenderingContext2D::DrawWindow(nsIDOMWindow* aWindow, PRInt32 aX, PRInt32 aY,
                                        PRInt32 aW, PRInt32 aH, 
@@ -2833,8 +2856,7 @@ nsCanvasRenderingContext2D::GetImageData()
     if (!JS_ConvertArguments (ctx, argc, argv, "jjjj", &x, &y, &w, &h))
         return NS_ERROR_DOM_SYNTAX_ERR;
 
-    if (w <= 0 || h <= 0 ||
-        x + w > mWidth || y + h > mHeight)
+    if (!CheckSaneSubrectSize (x, y, w, h, mWidth, mHeight))
         return NS_ERROR_DOM_SYNTAX_ERR;
 
     PRUint8 *surfaceData = mImageSurfaceData;
@@ -2965,7 +2987,7 @@ nsCanvasRenderingContext2D::PutImageData()
         return NS_ERROR_DOM_SYNTAX_ERR;
     dataArray = JSVAL_TO_OBJECT(v);
 
-    if (w <= 0 || h <= 0)
+    if (!CheckSaneSubrectSize (x, y, w, h, mWidth, mHeight))
         return NS_ERROR_DOM_SYNTAX_ERR;
 
     jsuint arrayLen;
@@ -2974,13 +2996,6 @@ nsCanvasRenderingContext2D::PutImageData()
         arrayLen < (jsuint)(w * h * 4))
         return NS_ERROR_DOM_SYNTAX_ERR;
 
-    // XXX I'm not sure if we really want this check -- we
-    // can just ignore any data that's set outside of the
-    // canvas boundaries
-    if (x + w > mWidth || y + h > mHeight)
-        return NS_ERROR_DOM_SYNTAX_ERR;
-
-
     nsAutoArrayPtr<PRUint8> imageBuffer(new PRUint8[w * h * 4]);
     cairo_surface_t *imgsurf;
     PRUint8 *imgPtr = imageBuffer.get();