From 87d1da1261c98e26ca58d0bbf25d047c60fd5d52 Mon Sep 17 00:00:00 2001 From: "bzbarsky%mit.edu" Date: Mon, 25 Aug 2008 15:16:31 +0000 Subject: [PATCH] Fix bug 316394 and bug 448166 by backporting the bug 316394 patch. git-svn-id: svn://10.0.0.236/branches/MOZILLA_1_8_BRANCH@253807 18797224-902f-48f8-a5cc-f745e15eee43 --- mozilla/gfx/src/gtk/nsFontMetricsGTK.cpp | 16 +++---- .../intl/unicharutil/util/nsUnicharUtils.h | 15 ++---- mozilla/layout/style/nsCSSScanner.cpp | 47 +++++++++---------- mozilla/layout/style/nsCSSScanner.h | 2 +- .../parser/htmlparser/src/nsHTMLTokens.cpp | 7 +-- mozilla/xpcom/string/public/nsCharTraits.h | 31 ++++++++++++ mozilla/xpcom/string/public/nsReadableUtils.h | 2 + mozilla/xpcom/string/public/nsUTF8Utils.h | 26 +++++----- mozilla/xpcom/string/src/nsReadableUtils.cpp | 16 +++++++ 9 files changed, 96 insertions(+), 66 deletions(-) diff --git a/mozilla/gfx/src/gtk/nsFontMetricsGTK.cpp b/mozilla/gfx/src/gtk/nsFontMetricsGTK.cpp index 0bdb4997bee..4a031e276ea 100644 --- a/mozilla/gfx/src/gtk/nsFontMetricsGTK.cpp +++ b/mozilla/gfx/src/gtk/nsFontMetricsGTK.cpp @@ -61,6 +61,7 @@ #include "nsAString.h" #include "nsXPIDLString.h" #include "nsFontDebug.h" +#include "nsCharTraits.h" #ifdef MOZ_ENABLE_FREETYPE2 #include "nsFT2FontNode.h" #include "nsFontFreeType.h" @@ -76,11 +77,8 @@ #include #include -#define IS_SURROGATE(u) (u > 0x10000) #define SAFE_CCMAP_HAS_CHAR_EXT(ccmap,c) ((ccmap) && ((IS_SURROGATE(c) && (ccmap)==gDoubleByteSpecialCharsCCMap) ? PR_FALSE : (CCMAP_HAS_CHAR_EXT(ccmap,c)))) -#define UCS2_NOMAPPING 0XFFFD - #ifdef PR_LOGGING static PRLogModuleInfo * FontMetricsGTKLM = PR_NewLogModule("FontMetricsGTK"); #endif /* PR_LOGGING */ @@ -3458,8 +3456,8 @@ nsFontMetricsGTK::PickASizeAndLoad(nsFontStretch* aStretch, return AddToLoadedFontsList(ftfont); } - if (IS_SURROGATE(aChar)) { - // SURROGATE is only supported by FreeType + if (!IS_IN_BMP(aChar)) { + // Non-BMP is only supported by FreeType return nsnull; } #endif @@ -5064,10 +5062,10 @@ nsFontMetricsGTK::SearchNode(nsFontNode* aNode, PRUint32 aChar) * loading a font with the same map. */ if (charSetInfo->mCharSet) { - // if SURROGATE char, ignore charSetInfo->mCCMap checking + // if not BMP char, ignore charSetInfo->mCCMap checking // because the exact ccmap is never created before loading // NEED TO FIX: need better way - if (IS_SURROGATE(aChar) ) { + if (!IS_IN_BMP(aChar) ) { goto check_done; } PRUint16* ccmap = charSetInfo->mCCMap; @@ -6561,8 +6559,8 @@ nsFontMetricsGTK::FindFont(PRUint32 aChar) // If this is is the 'unknown' char (ie: converter could not // convert it) there is no sense in searching any further for // a font. Just returing mWesternFont - if (aChar == UCS2_NOMAPPING) { - FIND_FONT_PRINTF((" ignore the 'UCS2_NOMAPPING' character, return mWesternFont")); + if (aChar == UCS2_REPLACEMENT_CHAR) { + FIND_FONT_PRINTF((" ignore the 'UCS2_REPLACEMENT_CHAR' character, return mWesternFont")); return mWesternFont; } diff --git a/mozilla/intl/unicharutil/util/nsUnicharUtils.h b/mozilla/intl/unicharutil/util/nsUnicharUtils.h index f021f1d953e..2407ed6a4b9 100644 --- a/mozilla/intl/unicharutil/util/nsUnicharUtils.h +++ b/mozilla/intl/unicharutil/util/nsUnicharUtils.h @@ -41,8 +41,13 @@ #ifndef nsAString_h___ #include "nsAString.h" #endif + #include "nsReadableUtils.h" +#ifndef nsCharTraits_h___ +#include "nsCharTraits.h" +#endif + void ToLowerCase( nsAString& ); void ToUpperCase( nsAString& ); @@ -90,16 +95,6 @@ inline PRBool IsLowerCase(PRUnichar c) { return ToUpperCase(c) != c; } -#define IS_HIGH_SURROGATE(u) ((PRUnichar)(u) >= (PRUnichar)0xd800 && (PRUnichar)(u) <= (PRUnichar)0xdbff) -#define IS_LOW_SURROGATE(u) ((PRUnichar)(u) >= (PRUnichar)0xdc00 && (PRUnichar)(u) <= (PRUnichar)0xdfff) - -#define SURROGATE_TO_UCS4(h, l) ((((PRUint32)(h)-(PRUint32)0xd800) << 10) + \ - (PRUint32)(l) - (PRUint32)(0xdc00) + 0x10000) - -#define H_SURROGATE(s) ((PRUnichar)(((PRUint32)s - (PRUint32)0x10000) >> 10) + (PRUnichar)0xd800) -#define L_SURROGATE(s) ((PRUnichar)(((PRUint32)s - (PRUint32)0x10000) & 0x3ff) + (PRUnichar)0xdc00) -#define IS_IN_BMP(ucs) ((PRUint32)ucs < 0x10000) - /* (0x3131u <= (u) && (u) <= 0x318eu) => Hangul Compatibility Jamo */ /* (0xac00u <= (u) && (u) <= 0xd7a3u) => Hangul Syllables */ #define IS_CJ_CHAR(u) \ diff --git a/mozilla/layout/style/nsCSSScanner.cpp b/mozilla/layout/style/nsCSSScanner.cpp index f8ca5f497c5..0f7e47d0a13 100644 --- a/mozilla/layout/style/nsCSSScanner.cpp +++ b/mozilla/layout/style/nsCSSScanner.cpp @@ -791,10 +791,7 @@ PRBool nsCSSScanner::NextURL(nsresult& aErrorCode, nsCSSToken& aToken) ch = Read(aErrorCode); if (ch < 0) break; if (ch == CSS_ESCAPE) { - ch = ParseEscape(aErrorCode); - if (0 < ch) { - ident.Append(PRUnichar(ch)); - } + ParseAndAppendEscape(aErrorCode, ident); } else if ((ch == '"') || (ch == '\'') || (ch == '(')) { // This is an invalid URL spec ok = PR_FALSE; @@ -830,12 +827,14 @@ PRBool nsCSSScanner::NextURL(nsresult& aErrorCode, nsCSSToken& aToken) } -PRInt32 nsCSSScanner::ParseEscape(nsresult& aErrorCode) +void +nsCSSScanner::ParseAndAppendEscape(nsresult& aErrorCode, nsString& aOutput) { PRUint8* lexTable = gLexTable; PRInt32 ch = Peek(aErrorCode); if (ch < 0) { - return CSS_ESCAPE; + aOutput.Append(CSS_ESCAPE); + return; } if ((ch <= 255) && ((lexTable[ch] & IS_HEX_DIGIT) != 0)) { PRInt32 rv = 0; @@ -882,18 +881,22 @@ PRInt32 nsCSSScanner::ParseEscape(nsresult& aErrorCode) } } } - return rv; + NS_ASSERTION(rv >= 0, "How did rv become negative?"); + if (rv > 0) { + AppendUCS4ToUTF16(ENSURE_VALID_CHAR(rv), aOutput); + } + return; } else { // "Any character except a hexidecimal digit can be escaped to // remove its special meaning by putting a backslash in front" // -- CSS1 spec section 7.1 - if (EatNewline(aErrorCode)) { // skip escaped newline - ch = 0; - } - else { + if (!EatNewline(aErrorCode)) { // skip escaped newline (void) Read(aErrorCode); + if (ch > 0) { + aOutput.Append(ch); + } } - return ch; + return; } } @@ -908,19 +911,16 @@ PRBool nsCSSScanner::GatherIdent(nsresult& aErrorCode, PRInt32 aChar, nsString& aIdent) { if (aChar == CSS_ESCAPE) { - aChar = ParseEscape(aErrorCode); + ParseAndAppendEscape(aErrorCode, aIdent); } - if (0 < aChar) { - aIdent.Append(PRUnichar(aChar)); + else if (0 < aChar) { + aIdent.Append(aChar); } for (;;) { aChar = Read(aErrorCode); if (aChar < 0) break; if (aChar == CSS_ESCAPE) { - aChar = ParseEscape(aErrorCode); - if (0 < aChar) { - aIdent.Append(PRUnichar(aChar)); - } + ParseAndAppendEscape(aErrorCode, aIdent); } else if ((aChar > 255) || ((gLexTable[aChar] & IS_IDENT) != 0)) { aIdent.Append(PRUnichar(aChar)); } else { @@ -1130,13 +1130,10 @@ PRBool nsCSSScanner::ParseString(nsresult& aErrorCode, PRInt32 aStop, break; } if (ch == CSS_ESCAPE) { - ch = ParseEscape(aErrorCode); - if (ch < 0) { - return PR_FALSE; - } + ParseAndAppendEscape(aErrorCode, aToken.mIdent); } - if (0 < ch) { - aToken.mIdent.Append(PRUnichar(ch)); + else if (0 < ch) { + aToken.mIdent.Append(ch); } } return PR_TRUE; diff --git a/mozilla/layout/style/nsCSSScanner.h b/mozilla/layout/style/nsCSSScanner.h index 68914006f82..2aaceef1da9 100644 --- a/mozilla/layout/style/nsCSSScanner.h +++ b/mozilla/layout/style/nsCSSScanner.h @@ -198,7 +198,7 @@ protected: PRBool EatWhiteSpace(nsresult& aErrorCode); PRBool EatNewline(nsresult& aErrorCode); - PRInt32 ParseEscape(nsresult& aErrorCode); + void ParseAndAppendEscape(nsresult& aErrorCode, nsString& aOutput); PRBool ParseIdent(nsresult& aErrorCode, PRInt32 aChar, nsCSSToken& aResult); PRBool ParseAtKeyword(nsresult& aErrorCode, PRInt32 aChar, nsCSSToken& aResult); diff --git a/mozilla/parser/htmlparser/src/nsHTMLTokens.cpp b/mozilla/parser/htmlparser/src/nsHTMLTokens.cpp index 3bb0c722578..698f016588e 100644 --- a/mozilla/parser/htmlparser/src/nsHTMLTokens.cpp +++ b/mozilla/parser/htmlparser/src/nsHTMLTokens.cpp @@ -2364,12 +2364,7 @@ static void AppendNCR(nsSubstring& aString, PRInt32 aNCRValue) } #endif - if (IS_IN_BMP(aNCRValue)) - aString.Append(PRUnichar(aNCRValue)); - else { - aString.Append(PRUnichar(H_SURROGATE(aNCRValue))); - aString.Append(PRUnichar(L_SURROGATE(aNCRValue))); - } + AppendUCS4ToUTF16(ENSURE_VALID_CHAR(aNCRValue), aString); } /* diff --git a/mozilla/xpcom/string/public/nsCharTraits.h b/mozilla/xpcom/string/public/nsCharTraits.h index e7713c9520f..49fa5fd4b64 100644 --- a/mozilla/xpcom/string/public/nsCharTraits.h +++ b/mozilla/xpcom/string/public/nsCharTraits.h @@ -73,6 +73,37 @@ typedef PRBool nsCharTraits_bool; #endif +// Some macros for working with PRUnichar +#define PLANE1_BASE PRUint32(0x00010000) +// High surrogates are in the range 0xD800 -- OxDBFF +#define IS_HIGH_SURROGATE(u) ((PRUnichar(u) & 0xFC00) == 0xD800) +// Low surrogates are in the range 0xDC00 -- 0xDFFF +#define IS_LOW_SURROGATE(u) ((PRUnichar(u) & 0xFC00) == 0xDC00) +// Faster than testing IS_HIGH_SURROGATE || IS_LOW_SURROGATE +#define IS_SURROGATE(u) ((PRUnichar(u) & 0xF800) == 0xD800) + +// Everything else is not a surrogate: 0x000 -- 0xD7FF, 0xE000 -- 0xFFFF + +// N = (H - 0xD800) * 0x400 + 0x10000 + (L - 0xDC00) +// I wonder whether we could somehow assert that H is a high surrogate +// and L is a low surrogate +#define SURROGATE_TO_UCS4(h, l) (((PRUint32(h) & 0x03FF) << 10) + \ + (PRUint32(l) & 0x03FF) + PLANE1_BASE) + +// Extract surrogates from a UCS4 char +// See unicode specification 3.7 for following math. +#define H_SURROGATE(c) PRUnichar(PRUnichar((PRUint32(c) - PLANE1_BASE) >> 10) | \ + PRUnichar(0xD800)) +#define L_SURROGATE(c) PRUnichar(PRUnichar((PRUint32(c) - PLANE1_BASE) & 0x03FF) | \ + PRUnichar(0xDC00)) + +#define IS_IN_BMP(ucs) (PRUint32(ucs) < PLANE1_BASE) +#define UCS2_REPLACEMENT_CHAR PRUnichar(0xFFFD) + +#define UCS_END PRUint32(0x00110000) +#define IS_VALID_CHAR(c) ((PRUint32(c) < UCS_END) && !IS_SURROGATE(c)) +#define ENSURE_VALID_CHAR(c) (IS_VALID_CHAR(c) ? (c) : UCS2_REPLACEMENT_CHAR) + template struct nsCharTraits {}; NS_SPECIALIZE_TEMPLATE diff --git a/mozilla/xpcom/string/public/nsReadableUtils.h b/mozilla/xpcom/string/public/nsReadableUtils.h index d3cda21e9da..b71d67db0aa 100755 --- a/mozilla/xpcom/string/public/nsReadableUtils.h +++ b/mozilla/xpcom/string/public/nsReadableUtils.h @@ -371,5 +371,7 @@ inline PRBool EnsureStringLength(T& aStr, PRUint32 aLen) return (aStr.Length() == aLen); } +NS_COM void +AppendUCS4ToUTF16(const PRUint32 aSource, nsAString& aDest); #endif // !defined(nsReadableUtils_h___) diff --git a/mozilla/xpcom/string/public/nsUTF8Utils.h b/mozilla/xpcom/string/public/nsUTF8Utils.h index f1e1b8b1aca..f6c634aa0d9 100644 --- a/mozilla/xpcom/string/public/nsUTF8Utils.h +++ b/mozilla/xpcom/string/public/nsUTF8Utils.h @@ -39,6 +39,8 @@ #ifndef nsUTF8Utils_h_ #define nsUTF8Utils_h_ +#include "nsCharTraits.h" + class UTF8traits { public: @@ -51,9 +53,6 @@ class UTF8traits static PRBool is6byte(char c) { return (c & 0xFE) == 0xFC; } }; -#define PLANE1_BASE 0x00010000 -#define UCS2_REPLACEMENT_CHAR 0xfffd - #ifdef __GNUC__ #define NS_ALWAYS_INLINE __attribute__((always_inline)) #else @@ -184,13 +183,11 @@ class ConvertUTF8toUTF16 } else if ( ucs4 >= PLANE1_BASE ) { - if ( ucs4 >= 0x00110000 ) + if ( ucs4 >= UCS_END ) *out++ = UCS2_REPLACEMENT_CHAR; else { - // surrogate, see unicode specification 3.7 for following math. - ucs4 -= PLANE1_BASE; - *out++ = (PRUnichar)(ucs4 >> 10) | 0xd800u; - *out++ = (PRUnichar)(ucs4 & 0x3ff) | 0xdc00u; + *out++ = (buffer_type)H_SURROGATE(ucs4); + *out++ = (buffer_type)L_SURROGATE(ucs4); } } else @@ -316,17 +313,16 @@ class ConvertUTF16toUTF8 *out++ = 0xC0 | (char)(c >> 6); *out++ = 0x80 | (char)(0x003F & c); } - else if (0xD800 != (0xF800 & c)) // U+0800 - U+D7FF,U+E000 - U+FFFF + else if (!IS_SURROGATE(c)) // U+0800 - U+D7FF,U+E000 - U+FFFF { *out++ = 0xE0 | (char)(c >> 12); *out++ = 0x80 | (char)(0x003F & (c >> 6)); *out++ = 0x80 | (char)(0x003F & c ); } - else if (0xD800 == (0xFC00 & c)) // U+D800 - U+DBFF + else if (IS_HIGH_SURROGATE(c)) // U+D800 - U+DBFF { // D800- DBFF - High Surrogate - // N = (H- D800) *400 + 10000 + ... - PRUint32 ucs4 = 0x10000 + ((0x03FF & c) << 10); + value_type h = c; ++p; if (p == end) @@ -337,11 +333,11 @@ class ConvertUTF16toUTF8 } c = *p; - if (0xDC00 == (0xFC00 & c)) + if (IS_LOW_SURROGATE(c)) { // DC00- DFFF - Low Surrogate - // N += ( L - DC00 ) - ucs4 |= (0x03FF & c); + // N = (H - D800) *400 + 10000 + ( L - DC00 ) + PRUint32 ucs4 = SURROGATE_TO_UCS4(h, c); // 0001 0000-001F FFFF *out++ = 0xF0 | (char)(ucs4 >> 18); diff --git a/mozilla/xpcom/string/src/nsReadableUtils.cpp b/mozilla/xpcom/string/src/nsReadableUtils.cpp index 3d56e4fd368..688bda2dd37 100755 --- a/mozilla/xpcom/string/src/nsReadableUtils.cpp +++ b/mozilla/xpcom/string/src/nsReadableUtils.cpp @@ -1094,3 +1094,19 @@ NS_COM const nsAFlatCString& EmptyCString() return sEmpty; } + +NS_COM +void +AppendUCS4ToUTF16(PRUint32 aSource, nsAString& aDest) + { + NS_ASSERTION(IS_VALID_CHAR(aSource), "Invalid UCS4 char"); + if (IS_IN_BMP(aSource)) + { + aDest.Append(PRUnichar(aSource)); + } + else + { + aDest.Append(H_SURROGATE(aSource)); + aDest.Append(L_SURROGATE(aSource)); + } + }