diff --git a/mozilla/htmlparser/src/nsScanner.cpp b/mozilla/htmlparser/src/nsScanner.cpp
index 0287ae5faaa..41a10b995e3 100644
--- a/mozilla/htmlparser/src/nsScanner.cpp
+++ b/mozilla/htmlparser/src/nsScanner.cpp
@@ -18,8 +18,12 @@
//#define __INCREMENTAL 1
+#define NS_IMPL_IDS
#include "nsScanner.h"
#include "nsDebug.h"
+#include "nsIServiceManager.h"
+#include "nsICharsetConverterManager.h"
+
const char* kBadHTMLText="
Oops...
You just tried to read a non-existent document:
";
const char* kUnorderedStringError = "String argument must be ordered. Don't you read API's?";
@@ -41,7 +45,7 @@ const int kBufsize=64;
* @return
*/
nsScanner::nsScanner(nsString& anHTMLString) :
- mBuffer(anHTMLString), mFilename("")
+ mBuffer(anHTMLString), mFilename("") , mCharset("")
{
mTotalRead=mBuffer.Length();
mIncremental=PR_TRUE;
@@ -49,6 +53,8 @@ nsScanner::nsScanner(nsString& anHTMLString) :
mOffset=0;
mMarkPos=-1;
mFileStream=0;
+ mUnicodeDecoder = nsnull;
+ InitUnicodeDecoder();
}
/**
@@ -61,7 +67,7 @@ nsScanner::nsScanner(nsString& anHTMLString) :
* @return
*/
nsScanner::nsScanner(nsString& aFilename,PRBool aCreateStream) :
- mBuffer(""), mFilename(aFilename)
+ mBuffer(""), mFilename(aFilename) , mCharset("")
{
mIncremental=PR_TRUE;
mOffset=0;
@@ -81,6 +87,9 @@ nsScanner::nsScanner(nsString& aFilename,PRBool aCreateStream) :
mFileStream=new fstream(buffer,ios::in);
#endif
} //if
+ mUnicodeDecoder = nsnull;
+ InitUnicodeDecoder();
+
}
/**
@@ -93,7 +102,7 @@ nsScanner::nsScanner(nsString& aFilename,PRBool aCreateStream) :
* @return
*/
nsScanner::nsScanner(nsString& aFilename,fstream& aStream,PRBool assumeOwnership) :
- mBuffer(""), mFilename(aFilename)
+ mBuffer(""), mFilename(aFilename) , mCharset("")
{
mIncremental=PR_TRUE;
mOffset=0;
@@ -101,6 +110,39 @@ nsScanner::nsScanner(nsString& aFilename,fstream& aStream,PRBool assumeOwnership
mTotalRead=0;
mOwnsStream=assumeOwnership;
mFileStream=&aStream;
+ mUnicodeDecoder = nsnull;
+ InitUnicodeDecoder();
+}
+
+void nsScanner::InitUnicodeDecoder()
+{
+ nsAutoString defaultCharset("ISO-8859-1");
+ SetDocumentCharset(defaultCharset);
+}
+nsresult nsScanner::SetDocumentCharset(nsString& aCharset )
+{
+ nsresult res = NS_OK;
+ if(! mCharset.EqualsIgnoreCase(aCharset)) // see do we need to change a converter.
+ {
+ nsICharsetConverterManager * ccm = nsnull;
+ res = nsServiceManager::GetService(kCharsetConverterManagerCID,
+ kICharsetConverterManagerIID,
+ (nsISupports**)&ccm);
+ if(NS_SUCCEEDED(res) && (nsnull != ccm))
+ {
+ nsIUnicodeDecoder * decoder = nsnull;
+ res = ccm->GetUnicodeDecoder(&aCharset, &decoder);
+ if(NS_SUCCEEDED(res) && (nsnull != decoder))
+ {
+ NS_IF_RELEASE(mUnicodeDecoder);
+
+ mUnicodeDecoder = decoder;
+ mCharset = aCharset;
+ }
+ nsServiceManager::ReleaseService(kCharsetConverterManagerCID, ccm);
+ }
+ }
+ return res;
}
@@ -118,6 +160,7 @@ nsScanner::~nsScanner() {
delete mFileStream;
}
mFileStream=0;
+ NS_IF_RELEASE(mUnicodeDecoder);
}
/**
@@ -176,8 +219,21 @@ PRBool nsScanner::Append(nsString& aBuffer) {
* @return
*/
PRBool nsScanner::Append(const char* aBuffer, PRUint32 aLen){
- mBuffer.Append(aBuffer,aLen);
- mTotalRead+=aLen;
+
+ PRInt32 unicharLength = 0;
+ PRInt32 srcLength = aLen;
+ mUnicodeDecoder->Length(aBuffer, 0, aLen, &unicharLength);
+ PRUnichar *unichars = new PRUnichar [ unicharLength ];
+
+ nsresult res = mUnicodeDecoder->Convert(unichars, 0, &unicharLength,
+ aBuffer, 0, &srcLength );
+ mBuffer.Append(unichars, unicharLength);
+ delete unichars;
+ mTotalRead += unicharLength;
+
+ // mBuffer.Append(aBuffer,aLen);
+ // mTotalRead+=aLen;
+
return PR_TRUE;
}
diff --git a/mozilla/htmlparser/src/nsScanner.h b/mozilla/htmlparser/src/nsScanner.h
index 58004722d29..c27e400b24f 100644
--- a/mozilla/htmlparser/src/nsScanner.h
+++ b/mozilla/htmlparser/src/nsScanner.h
@@ -35,6 +35,7 @@
#include "nsString.h"
#include "nsIParser.h"
#include "prtypes.h"
+#include "nsIUnicodeDecoder.h"
#include
@@ -265,7 +266,13 @@ class nsScanner {
static void SelfTest();
+ /**
+ * Change the charset and the Unicode Decoder
+ */
+ nsresult SetDocumentCharset(nsString& aCharset);
+
protected:
+
enum {eBufferSizeThreshold=512};
/**
@@ -276,6 +283,10 @@ class nsScanner {
*/
nsresult FillBuffer(void);
+ /**
+ * Initialize the charset and the Unicode Decoder
+ */
+ void InitUnicodeDecoder();
fstream* mFileStream;
nsString mBuffer;
@@ -285,6 +296,8 @@ class nsScanner {
PRUint32 mTotalRead;
PRBool mOwnsStream;
PRBool mIncremental;
+ nsString mCharset;
+ nsIUnicodeDecoder *mUnicodeDecoder;
};
#endif
diff --git a/mozilla/parser/htmlparser/src/nsScanner.cpp b/mozilla/parser/htmlparser/src/nsScanner.cpp
index 0287ae5faaa..41a10b995e3 100644
--- a/mozilla/parser/htmlparser/src/nsScanner.cpp
+++ b/mozilla/parser/htmlparser/src/nsScanner.cpp
@@ -18,8 +18,12 @@
//#define __INCREMENTAL 1
+#define NS_IMPL_IDS
#include "nsScanner.h"
#include "nsDebug.h"
+#include "nsIServiceManager.h"
+#include "nsICharsetConverterManager.h"
+
const char* kBadHTMLText="Oops...
You just tried to read a non-existent document:
";
const char* kUnorderedStringError = "String argument must be ordered. Don't you read API's?";
@@ -41,7 +45,7 @@ const int kBufsize=64;
* @return
*/
nsScanner::nsScanner(nsString& anHTMLString) :
- mBuffer(anHTMLString), mFilename("")
+ mBuffer(anHTMLString), mFilename("") , mCharset("")
{
mTotalRead=mBuffer.Length();
mIncremental=PR_TRUE;
@@ -49,6 +53,8 @@ nsScanner::nsScanner(nsString& anHTMLString) :
mOffset=0;
mMarkPos=-1;
mFileStream=0;
+ mUnicodeDecoder = nsnull;
+ InitUnicodeDecoder();
}
/**
@@ -61,7 +67,7 @@ nsScanner::nsScanner(nsString& anHTMLString) :
* @return
*/
nsScanner::nsScanner(nsString& aFilename,PRBool aCreateStream) :
- mBuffer(""), mFilename(aFilename)
+ mBuffer(""), mFilename(aFilename) , mCharset("")
{
mIncremental=PR_TRUE;
mOffset=0;
@@ -81,6 +87,9 @@ nsScanner::nsScanner(nsString& aFilename,PRBool aCreateStream) :
mFileStream=new fstream(buffer,ios::in);
#endif
} //if
+ mUnicodeDecoder = nsnull;
+ InitUnicodeDecoder();
+
}
/**
@@ -93,7 +102,7 @@ nsScanner::nsScanner(nsString& aFilename,PRBool aCreateStream) :
* @return
*/
nsScanner::nsScanner(nsString& aFilename,fstream& aStream,PRBool assumeOwnership) :
- mBuffer(""), mFilename(aFilename)
+ mBuffer(""), mFilename(aFilename) , mCharset("")
{
mIncremental=PR_TRUE;
mOffset=0;
@@ -101,6 +110,39 @@ nsScanner::nsScanner(nsString& aFilename,fstream& aStream,PRBool assumeOwnership
mTotalRead=0;
mOwnsStream=assumeOwnership;
mFileStream=&aStream;
+ mUnicodeDecoder = nsnull;
+ InitUnicodeDecoder();
+}
+
+void nsScanner::InitUnicodeDecoder()
+{
+ nsAutoString defaultCharset("ISO-8859-1");
+ SetDocumentCharset(defaultCharset);
+}
+nsresult nsScanner::SetDocumentCharset(nsString& aCharset )
+{
+ nsresult res = NS_OK;
+ if(! mCharset.EqualsIgnoreCase(aCharset)) // see do we need to change a converter.
+ {
+ nsICharsetConverterManager * ccm = nsnull;
+ res = nsServiceManager::GetService(kCharsetConverterManagerCID,
+ kICharsetConverterManagerIID,
+ (nsISupports**)&ccm);
+ if(NS_SUCCEEDED(res) && (nsnull != ccm))
+ {
+ nsIUnicodeDecoder * decoder = nsnull;
+ res = ccm->GetUnicodeDecoder(&aCharset, &decoder);
+ if(NS_SUCCEEDED(res) && (nsnull != decoder))
+ {
+ NS_IF_RELEASE(mUnicodeDecoder);
+
+ mUnicodeDecoder = decoder;
+ mCharset = aCharset;
+ }
+ nsServiceManager::ReleaseService(kCharsetConverterManagerCID, ccm);
+ }
+ }
+ return res;
}
@@ -118,6 +160,7 @@ nsScanner::~nsScanner() {
delete mFileStream;
}
mFileStream=0;
+ NS_IF_RELEASE(mUnicodeDecoder);
}
/**
@@ -176,8 +219,21 @@ PRBool nsScanner::Append(nsString& aBuffer) {
* @return
*/
PRBool nsScanner::Append(const char* aBuffer, PRUint32 aLen){
- mBuffer.Append(aBuffer,aLen);
- mTotalRead+=aLen;
+
+ PRInt32 unicharLength = 0;
+ PRInt32 srcLength = aLen;
+ mUnicodeDecoder->Length(aBuffer, 0, aLen, &unicharLength);
+ PRUnichar *unichars = new PRUnichar [ unicharLength ];
+
+ nsresult res = mUnicodeDecoder->Convert(unichars, 0, &unicharLength,
+ aBuffer, 0, &srcLength );
+ mBuffer.Append(unichars, unicharLength);
+ delete unichars;
+ mTotalRead += unicharLength;
+
+ // mBuffer.Append(aBuffer,aLen);
+ // mTotalRead+=aLen;
+
return PR_TRUE;
}
diff --git a/mozilla/parser/htmlparser/src/nsScanner.h b/mozilla/parser/htmlparser/src/nsScanner.h
index 58004722d29..c27e400b24f 100644
--- a/mozilla/parser/htmlparser/src/nsScanner.h
+++ b/mozilla/parser/htmlparser/src/nsScanner.h
@@ -35,6 +35,7 @@
#include "nsString.h"
#include "nsIParser.h"
#include "prtypes.h"
+#include "nsIUnicodeDecoder.h"
#include
@@ -265,7 +266,13 @@ class nsScanner {
static void SelfTest();
+ /**
+ * Change the charset and the Unicode Decoder
+ */
+ nsresult SetDocumentCharset(nsString& aCharset);
+
protected:
+
enum {eBufferSizeThreshold=512};
/**
@@ -276,6 +283,10 @@ class nsScanner {
*/
nsresult FillBuffer(void);
+ /**
+ * Initialize the charset and the Unicode Decoder
+ */
+ void InitUnicodeDecoder();
fstream* mFileStream;
nsString mBuffer;
@@ -285,6 +296,8 @@ class nsScanner {
PRUint32 mTotalRead;
PRBool mOwnsStream;
PRBool mIncremental;
+ nsString mCharset;
+ nsIUnicodeDecoder *mUnicodeDecoder;
};
#endif