Bommels05/Mozilla
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug91437 - Speeding up the tokenizer slightly by using a smarter scan algorithm and avoiding trying to skip whitespace when there are none. r=harishd sr=jst

Browse Source 
git-svn-id: svn://10.0.0.236/trunk@101192 18797224-902f-48f8-a5cc-f745e15eee43
This commit is contained in:
bratell%lysator.liu.se
2001-08-16 05:24:17 +00:00
parent 2d23b9911a
commit d4bca51618
6 changed files with 320 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
mozilla/htmlparser/src/nsHTMLTokens.cpp
View File

@@ -536,8 +536,7 @@ nsresult CTextToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 aFlag)
static const PRUnichar theTerminalsChars[] =
{ PRUnichar('\n'), PRUnichar('\r'), PRUnichar('&'), PRUnichar('<'),
PRUnichar(0) };
const nsDependentString theTerminals(theTerminalsChars,
sizeof(theTerminalsChars)/sizeof(theTerminalsChars[0]) - 1);
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsresult result=NS_OK;
PRBool done=PR_FALSE;
nsReadingIterator<PRUnichar> origin, start, end;
@@ -550,7 +549,7 @@ nsresult CTextToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 aFlag)
aScanner.SetPosition(start);
while((NS_OK==result) && (!done)) {
result=aScanner.ReadUntil(start, end, theTerminals,PR_FALSE);
result=aScanner.ReadUntil(start, end, theEndCondition, PR_FALSE);
if(NS_OK==result) {
result=aScanner.Peek(aChar);
@@ -792,12 +791,14 @@ PRInt32 CCDATASectionToken::GetTokenType(void) {
* @return error result
*/
nsresult CCDATASectionToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 aFlag) {
static const char* theTerminals="\r]";
static const PRUnichar theTerminalsChars[] =
{ PRUnichar('\r'), PRUnichar(']'), PRUnichar(0) };
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsresult result=NS_OK;
PRBool done=PR_FALSE;
while((NS_OK==result) && (!done)) {
result=aScanner.ReadUntil(mTextValue,theTerminals,PR_FALSE);
result=aScanner.ReadUntil(mTextValue,theEndCondition,PR_FALSE);
if(NS_OK==result) {
result=aScanner.Peek(aChar);
if((kCR==aChar) && (NS_OK==result)) {
@@ -898,8 +899,7 @@ nsresult CMarkupDeclToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32
{ PRUnichar('\n'), PRUnichar('\r'), PRUnichar('\''), PRUnichar('"'),
PRUnichar('>'),
PRUnichar(0) };
const nsDependentString theTerminals(theTerminalsChars,
sizeof(theTerminalsChars)/sizeof(theTerminalsChars[0]) - 1);
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsresult result=NS_OK;
PRBool done=PR_FALSE;
PRUnichar quote=0;
@@ -910,7 +910,7 @@ nsresult CMarkupDeclToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32
while((NS_OK==result) && (!done)) {
aScanner.SetPosition(start);
result=aScanner.ReadUntil(start, end, theTerminals,PR_FALSE);
result=aScanner.ReadUntil(start, end, theEndCondition, PR_FALSE);
if(NS_OK==result) {
result=aScanner.Peek(aChar);
@@ -1017,6 +1017,9 @@ CCommentToken::CCommentToken(const nsAReadableString& aName) : CHTMLToken(eHTMLT
static
nsresult ConsumeStrictComment(PRUnichar aChar, nsScanner& aScanner,nsString& aString) {
nsresult result=NS_OK;
static const PRUnichar theTerminalsChars[] =
{ PRUnichar('-'), PRUnichar('>'), PRUnichar(0) };
static const nsReadEndCondition endCommentCondition(theTerminalsChars);
/*********************************************************
NOTE: This algorithm does a fine job of handling comments
@@ -1049,8 +1052,8 @@ nsresult ConsumeStrictComment(PRUnichar aChar, nsScanner& aScanner,nsString& aSt
aString+=temp;
if(NS_OK==result) {
if(NS_OK==result) {
temp.AssignWithConversion("->");
result=aScanner.ReadUntil(aString,temp,PR_FALSE);
// Read until "-" or ">"
result=aScanner.ReadUntil(aString,endCommentCondition,PR_FALSE);
}
}
}
@@ -1540,13 +1543,10 @@ nsresult ConsumeAttributeEntity(nsString& aString,
static
nsresult ConsumeAttributeValueText(nsString& aString,
nsScanner& aScanner,
const PRUnichar *aTerminalChars,
const nsReadEndCondition& aEndCondition,
PRInt32 aFlag)
{
const nsDependentString theTerminals(aTerminalChars,
sizeof(aTerminalChars)/sizeof(aTerminalChars[0]) - 1);
nsresult result=aScanner.ReadUntil(aString,theTerminals,PR_FALSE);
nsresult result=aScanner.ReadUntil(aString,aEndCondition,PR_FALSE);
if(NS_SUCCEEDED(result)) {
PRUnichar ch;
@@ -1554,7 +1554,7 @@ nsresult ConsumeAttributeValueText(nsString& aString,
if(ch==kAmpersand) {
result=ConsumeAttributeEntity(aString,aScanner,aFlag);
if (NS_SUCCEEDED(result)) {
result=ConsumeAttributeValueText(aString,aScanner,aTerminalChars,aFlag);
result=ConsumeAttributeValueText(aString,aScanner,aEndCondition,aFlag);
}
}
}
@@ -1579,16 +1579,25 @@ nsresult ConsumeQuotedString(PRUnichar aChar,
{
NS_ASSERTION(aChar==kQuote || aChar==kApostrophe,"char is neither quote nor apostrophe");
const PRUnichar theTerminalChars[] = {
aChar, PRUnichar('&'),
PRUnichar(0)
};
static const PRUnichar theTerminalCharsQuote[] = {
PRUnichar(kQuote), PRUnichar('&'), PRUnichar(0) };
static const PRUnichar theTerminalCharsApostrophe[] = {
PRUnichar(kApostrophe), PRUnichar('&'), PRUnichar(0) };
static const nsReadEndCondition
theTerminateConditionQuote(theTerminalCharsQuote);
static const nsReadEndCondition
theTerminateConditionApostrophe(theTerminalCharsApostrophe);
// Assume Quote to init to something
const nsReadEndCondition *terminateCondition = &theTerminateConditionQuote;
if (aChar==kApostrophe)
terminateCondition = &theTerminateConditionApostrophe;
nsresult result=NS_OK;
nsReadingIterator<PRUnichar> theOffset;
aScanner.CurrentPosition(theOffset);
result=ConsumeAttributeValueText(aString,aScanner,theTerminalChars,aFlag);
result=ConsumeAttributeValueText(aString,aScanner,*terminateCondition,aFlag);
if(NS_SUCCEEDED(result)) {
result = aScanner.SkipOver(aChar); // aChar should be " or '
@@ -1599,9 +1608,11 @@ nsresult ConsumeQuotedString(PRUnichar aChar,
// Ex <table> <tr d="><td>hello</td></tr></table>
if(!aString.IsEmpty() && aString.Last()!=aChar &&
!aScanner.IsIncremental() && result==kEOF) {
static const nsReadEndCondition
theAttributeTerminator(kAttributeTerminalChars);
aString.Truncate();
aScanner.SetPosition(theOffset, PR_FALSE, PR_TRUE);
result=ConsumeAttributeValueText(aString,aScanner,kAttributeTerminalChars,aFlag);
result=ConsumeAttributeValueText(aString,aScanner,theAttributeTerminator,aFlag);
}
return result;
}
@@ -1638,11 +1649,10 @@ nsresult CAttributeToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 a
PRUnichar('\r'), PRUnichar('\t'),
PRUnichar('>'), PRUnichar('\b'),
PRUnichar(0) };
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsReadingIterator<PRUnichar> start, end;
const nsDependentString theTerminals(theTerminalsChars,
sizeof(theTerminalsChars)/sizeof(theTerminalsChars[0]) - 1);
result=aScanner.ReadUntil(start,end,theTerminals,PR_FALSE);
result=aScanner.ReadUntil(start,end,theEndCondition,PR_FALSE);
if (!(aFlag & NS_IPARSER_FLAG_VIEW_SOURCE)) {
aScanner.BindSubstring(mTextKey, start, end);
@@ -1694,9 +1704,11 @@ nsresult CAttributeToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 a
mHasEqualWithoutValue=PR_TRUE;
}
else {
static const nsReadEndCondition
theAttributeTerminator(kAttributeTerminalChars);
result=ConsumeAttributeValueText(mTextValue,
aScanner,
kAttributeTerminalChars,
theAttributeTerminator,
aFlag);
}
}//if
@@ -2342,16 +2354,14 @@ nsresult CDoctypeDeclToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32
{ PRUnichar('>'), PRUnichar('<'),
PRUnichar(0)
};
const nsDependentString terminals(terminalChars,
sizeof(terminalChars)/sizeof(terminalChars[0]) - 1);
static const nsReadEndCondition theEndCondition(terminalChars);
nsReadingIterator<PRUnichar> start, end;
aScanner.CurrentPosition(start);
aScanner.EndReading(end);
nsresult result=aScanner.ReadUntil(start, end, terminals,PR_FALSE);
nsresult result=aScanner.ReadUntil(start, end, theEndCondition, PR_FALSE);
if (NS_SUCCEEDED(result)) {
PRUnichar ch;

240
mozilla/htmlparser/src/nsScanner.cpp
View File

@@ -48,6 +48,25 @@ nsScannerString::ReplaceCharacter(nsReadingIterator<PRUnichar>& aPosition,
*pos = aChar;
}
nsReadEndCondition::nsReadEndCondition(const PRUnichar* aTerminateChars) :
mChars(aTerminateChars), mFilter(PRUnichar(~0)) // All bits set
{
// Build filter that will be used to filter out characters with
// bits that none of the terminal chars have. This works very well
// because terminal chars often have only the last 4-6 bits set and
// normal ascii letters have bit 7 set. Other letters have even higher
// bits set.
// Calculate filter
const PRUnichar *current = aTerminateChars;
PRUnichar terminalChar = *current;
while (terminalChar) {
mFilter &= ~terminalChar;
++current;
terminalChar = *current;
}
}
static NS_DEFINE_CID(kCharsetConverterManagerCID, NS_ICHARSETCONVERTERMANAGER_CID);
const char* kBadHTMLText="<H3>Oops...</H3>You just tried to read a non-existent document: <BR>";
@@ -533,17 +552,18 @@ nsresult nsScanner::SkipWhitespace(void) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> current, end;
PRBool found=PR_FALSE;
nsReadingIterator<PRUnichar> current;
PRBool found;
PRBool skipped = PR_FALSE;
mNewlinesSkipped = 0;
current = mCurrentPosition;
end = mEndPosition;
while(current != end) {
theChar=*current;
PRUnichar theChar=0;
nsresult result=Peek(theChar);
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
switch(theChar) {
case '\n': mNewlinesSkipped++;
case ' ' :
@@ -559,19 +579,22 @@ nsresult nsScanner::SkipWhitespace(void) {
if(!found) {
break;
}
else {
++current;
}
++current;
theChar = *current;
skipped = PR_TRUE;
}
SetPosition(current);
if (current == end) {
if (!skipped)
return NS_OK;
if (current == mEndPosition) {
SetPosition(current);
return Eof();
}
//DoErrTest(aString);
SetPosition(current);
return result;
return NS_OK;
}
@@ -1156,156 +1179,113 @@ nsresult nsScanner::ReadWhile(nsString& aString,
* @return error code
*/
nsresult nsScanner::ReadUntil(nsAWritableString& aString,
const nsAFlatString& aTerminalSet,
const nsReadEndCondition& aEndCondition,
PRBool addTerminal)
{
if (!mSlidingBuffer) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
const PRUnichar* setstart = aTerminalSet.get();
nsReadingIterator<PRUnichar> origin, current;
const PRUnichar* setstart = aEndCondition.mChars;
const PRUnichar* setcurrent;
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
setcurrent = setstart;
theChar=*current;
if(theChar) {
while (*setcurrent) {
if (*setcurrent == theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
goto found;
}
++setcurrent;
}
}
++current;
}
found:
SetPosition(current);
if (current == end) {
AppendUnicodeTo(origin, current, aString);
return Eof();
}
//DoErrTest(aString);
return result;
}
/**
* Consume characters until you encounter one contained in given
* input set.
*
* @update gess 3/25/98
* @param aString will contain the result of this method
* @param aTerminalSet is an ordered string that contains
* the set of INVALID characters
* @return error code
*/
nsresult nsScanner::ReadUntil(nsAWritableString& aString,
const nsAFlatCString& aTerminalSet,
PRBool addTerminal)
{
if (!mSlidingBuffer) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
const char* setstart = aTerminalSet.get();
const char* setcurrent;
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
// Filter out completely wrong characters
// Check if all bits are in the required area
if(!(theChar & aEndCondition.mFilter)) {
// They were. Do a thorough check.
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
setcurrent = setstart;
theChar=*current;
if(theChar) {
setcurrent = setstart;
while (*setcurrent) {
if (*setcurrent == theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
goto found;
}
++setcurrent;
}
}
++current;
theChar = *current;
}
found:
// If we are here, we didn't find any terminator in the string and
// current = mEndPosition
SetPosition(current);
if (current == end) {
AppendUnicodeTo(origin, current, aString);
return Eof();
}
AppendUnicodeTo(origin, current, aString);
return Eof();
found:
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
SetPosition(current);
//DoErrTest(aString);
return result;
return NS_OK;
}
nsresult nsScanner::ReadUntil(nsReadingIterator<PRUnichar>& aStart,
nsReadingIterator<PRUnichar>& aEnd,
const nsAFlatString& aTerminalSet,
const nsReadEndCondition &aEndCondition,
PRBool addTerminal)
{
if (!mSlidingBuffer) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
const PRUnichar* setstart = aTerminalSet.get();
nsReadingIterator<PRUnichar> origin, current;
const PRUnichar* setstart = aEndCondition.mChars;
const PRUnichar* setcurrent;
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
setcurrent = setstart;
theChar=*current;
if(theChar) {
PRUnichar theChar=0;
nsresult result=Peek(theChar);
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
// Filter out completely wrong characters
// Check if all bits are in the required area
if(!(theChar & aEndCondition.mFilter)) {
// They were. Do a thorough check.
setcurrent = setstart;
while (*setcurrent) {
if (*setcurrent == theChar) {
if(addTerminal)
++current;
aStart = origin;
aEnd = current;
goto found;
}
++setcurrent;
++setcurrent;
}
}
++current;
theChar = *current;
}
found:
// If we are here, we didn't find any terminator in the string and
// current = mEndPosition
SetPosition(current);
if (current == end) {
aStart = origin;
aEnd = current;
return Eof();
}
aStart = origin;
aEnd = current;
return Eof();
return result;
found:
if(addTerminal)
++current;
aStart = origin;
aEnd = current;
SetPosition(current);
return NS_OK;
}
/**
@@ -1323,36 +1303,32 @@ nsresult nsScanner::ReadUntil(nsAWritableString& aString,
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
nsReadingIterator<PRUnichar> origin, current;
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
theChar=*current;
if(theChar) {
if(aTerminalChar==theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
break;
}
PRUnichar theChar;
nsresult result=Peek(theChar);
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
if (aTerminalChar == theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
SetPosition(current);
return NS_OK;
}
++current;
theChar = *current;
}
// If we are here, we didn't find any terminator in the string and
// current = mEndPosition
AppendUnicodeTo(origin, current, aString);
SetPosition(current);
if (current == end) {
AppendUnicodeTo(origin, current, aString);
return Eof();
}
//DoErrTest(aString);
return result;
return Eof();
}

27
mozilla/htmlparser/src/nsScanner.h
View File

@@ -54,6 +54,16 @@ class nsScannerString : public nsSlidingString {
PRUnichar aChar);
};
class nsReadEndCondition {
public:
const PRUnichar *mChars;
PRUnichar mFilter;
explicit nsReadEndCondition(const PRUnichar* aTerminateChars);
private:
nsReadEndCondition(const nsReadEndCondition& aOther); // No copying
void operator=(const nsReadEndCondition& aOther); // No assigning
};
class nsScanner {
public:
@@ -216,25 +226,12 @@ class nsScanner {
* @return error code
*/
nsresult ReadUntil(nsAWritableString& aString,
const nsAFlatString& aTermSet,
const nsReadEndCondition& aEndCondition,
PRBool addTerminal);
nsresult ReadUntil(nsAWritableString& aString,
const nsAFlatCString& aTermSet,
PRBool addTerminal);
nsresult ReadUntil(nsAWritableString& aString,
const char* aTerminalSet,
PRBool addTerminal)
{
return ReadUntil(aString,
nsDependentCString(aTerminalSet),
addTerminal);
}
nsresult ReadUntil(nsReadingIterator<PRUnichar>& aStart,
nsReadingIterator<PRUnichar>& aEnd,
const nsAFlatString& aTerminalSet,
const nsReadEndCondition& aEndCondition,
PRBool addTerminal);

70
mozilla/parser/htmlparser/src/nsHTMLTokens.cpp
View File

@@ -536,8 +536,7 @@ nsresult CTextToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 aFlag)
static const PRUnichar theTerminalsChars[] =
{ PRUnichar('\n'), PRUnichar('\r'), PRUnichar('&'), PRUnichar('<'),
PRUnichar(0) };
const nsDependentString theTerminals(theTerminalsChars,
sizeof(theTerminalsChars)/sizeof(theTerminalsChars[0]) - 1);
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsresult result=NS_OK;
PRBool done=PR_FALSE;
nsReadingIterator<PRUnichar> origin, start, end;
@@ -550,7 +549,7 @@ nsresult CTextToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 aFlag)
aScanner.SetPosition(start);
while((NS_OK==result) && (!done)) {
result=aScanner.ReadUntil(start, end, theTerminals,PR_FALSE);
result=aScanner.ReadUntil(start, end, theEndCondition, PR_FALSE);
if(NS_OK==result) {
result=aScanner.Peek(aChar);
@@ -792,12 +791,14 @@ PRInt32 CCDATASectionToken::GetTokenType(void) {
* @return error result
*/
nsresult CCDATASectionToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 aFlag) {
static const char* theTerminals="\r]";
static const PRUnichar theTerminalsChars[] =
{ PRUnichar('\r'), PRUnichar(']'), PRUnichar(0) };
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsresult result=NS_OK;
PRBool done=PR_FALSE;
while((NS_OK==result) && (!done)) {
result=aScanner.ReadUntil(mTextValue,theTerminals,PR_FALSE);
result=aScanner.ReadUntil(mTextValue,theEndCondition,PR_FALSE);
if(NS_OK==result) {
result=aScanner.Peek(aChar);
if((kCR==aChar) && (NS_OK==result)) {
@@ -898,8 +899,7 @@ nsresult CMarkupDeclToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32
{ PRUnichar('\n'), PRUnichar('\r'), PRUnichar('\''), PRUnichar('"'),
PRUnichar('>'),
PRUnichar(0) };
const nsDependentString theTerminals(theTerminalsChars,
sizeof(theTerminalsChars)/sizeof(theTerminalsChars[0]) - 1);
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsresult result=NS_OK;
PRBool done=PR_FALSE;
PRUnichar quote=0;
@@ -910,7 +910,7 @@ nsresult CMarkupDeclToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32
while((NS_OK==result) && (!done)) {
aScanner.SetPosition(start);
result=aScanner.ReadUntil(start, end, theTerminals,PR_FALSE);
result=aScanner.ReadUntil(start, end, theEndCondition, PR_FALSE);
if(NS_OK==result) {
result=aScanner.Peek(aChar);
@@ -1017,6 +1017,9 @@ CCommentToken::CCommentToken(const nsAReadableString& aName) : CHTMLToken(eHTMLT
static
nsresult ConsumeStrictComment(PRUnichar aChar, nsScanner& aScanner,nsString& aString) {
nsresult result=NS_OK;
static const PRUnichar theTerminalsChars[] =
{ PRUnichar('-'), PRUnichar('>'), PRUnichar(0) };
static const nsReadEndCondition endCommentCondition(theTerminalsChars);
/*********************************************************
NOTE: This algorithm does a fine job of handling comments
@@ -1049,8 +1052,8 @@ nsresult ConsumeStrictComment(PRUnichar aChar, nsScanner& aScanner,nsString& aSt
aString+=temp;
if(NS_OK==result) {
if(NS_OK==result) {
temp.AssignWithConversion("->");
result=aScanner.ReadUntil(aString,temp,PR_FALSE);
// Read until "-" or ">"
result=aScanner.ReadUntil(aString,endCommentCondition,PR_FALSE);
}
}
}
@@ -1540,13 +1543,10 @@ nsresult ConsumeAttributeEntity(nsString& aString,
static
nsresult ConsumeAttributeValueText(nsString& aString,
nsScanner& aScanner,
const PRUnichar *aTerminalChars,
const nsReadEndCondition& aEndCondition,
PRInt32 aFlag)
{
const nsDependentString theTerminals(aTerminalChars,
sizeof(aTerminalChars)/sizeof(aTerminalChars[0]) - 1);
nsresult result=aScanner.ReadUntil(aString,theTerminals,PR_FALSE);
nsresult result=aScanner.ReadUntil(aString,aEndCondition,PR_FALSE);
if(NS_SUCCEEDED(result)) {
PRUnichar ch;
@@ -1554,7 +1554,7 @@ nsresult ConsumeAttributeValueText(nsString& aString,
if(ch==kAmpersand) {
result=ConsumeAttributeEntity(aString,aScanner,aFlag);
if (NS_SUCCEEDED(result)) {
result=ConsumeAttributeValueText(aString,aScanner,aTerminalChars,aFlag);
result=ConsumeAttributeValueText(aString,aScanner,aEndCondition,aFlag);
}
}
}
@@ -1579,16 +1579,25 @@ nsresult ConsumeQuotedString(PRUnichar aChar,
{
NS_ASSERTION(aChar==kQuote || aChar==kApostrophe,"char is neither quote nor apostrophe");
const PRUnichar theTerminalChars[] = {
aChar, PRUnichar('&'),
PRUnichar(0)
};
static const PRUnichar theTerminalCharsQuote[] = {
PRUnichar(kQuote), PRUnichar('&'), PRUnichar(0) };
static const PRUnichar theTerminalCharsApostrophe[] = {
PRUnichar(kApostrophe), PRUnichar('&'), PRUnichar(0) };
static const nsReadEndCondition
theTerminateConditionQuote(theTerminalCharsQuote);
static const nsReadEndCondition
theTerminateConditionApostrophe(theTerminalCharsApostrophe);
// Assume Quote to init to something
const nsReadEndCondition *terminateCondition = &theTerminateConditionQuote;
if (aChar==kApostrophe)
terminateCondition = &theTerminateConditionApostrophe;
nsresult result=NS_OK;
nsReadingIterator<PRUnichar> theOffset;
aScanner.CurrentPosition(theOffset);
result=ConsumeAttributeValueText(aString,aScanner,theTerminalChars,aFlag);
result=ConsumeAttributeValueText(aString,aScanner,*terminateCondition,aFlag);
if(NS_SUCCEEDED(result)) {
result = aScanner.SkipOver(aChar); // aChar should be " or '
@@ -1599,9 +1608,11 @@ nsresult ConsumeQuotedString(PRUnichar aChar,
// Ex <table> <tr d="><td>hello</td></tr></table>
if(!aString.IsEmpty() && aString.Last()!=aChar &&
!aScanner.IsIncremental() && result==kEOF) {
static const nsReadEndCondition
theAttributeTerminator(kAttributeTerminalChars);
aString.Truncate();
aScanner.SetPosition(theOffset, PR_FALSE, PR_TRUE);
result=ConsumeAttributeValueText(aString,aScanner,kAttributeTerminalChars,aFlag);
result=ConsumeAttributeValueText(aString,aScanner,theAttributeTerminator,aFlag);
}
return result;
}
@@ -1638,11 +1649,10 @@ nsresult CAttributeToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 a
PRUnichar('\r'), PRUnichar('\t'),
PRUnichar('>'), PRUnichar('\b'),
PRUnichar(0) };
static const nsReadEndCondition theEndCondition(theTerminalsChars);
nsReadingIterator<PRUnichar> start, end;
const nsDependentString theTerminals(theTerminalsChars,
sizeof(theTerminalsChars)/sizeof(theTerminalsChars[0]) - 1);
result=aScanner.ReadUntil(start,end,theTerminals,PR_FALSE);
result=aScanner.ReadUntil(start,end,theEndCondition,PR_FALSE);
if (!(aFlag & NS_IPARSER_FLAG_VIEW_SOURCE)) {
aScanner.BindSubstring(mTextKey, start, end);
@@ -1694,9 +1704,11 @@ nsresult CAttributeToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32 a
mHasEqualWithoutValue=PR_TRUE;
}
else {
static const nsReadEndCondition
theAttributeTerminator(kAttributeTerminalChars);
result=ConsumeAttributeValueText(mTextValue,
aScanner,
kAttributeTerminalChars,
theAttributeTerminator,
aFlag);
}
}//if
@@ -2342,16 +2354,14 @@ nsresult CDoctypeDeclToken::Consume(PRUnichar aChar, nsScanner& aScanner,PRInt32
{ PRUnichar('>'), PRUnichar('<'),
PRUnichar(0)
};
const nsDependentString terminals(terminalChars,
sizeof(terminalChars)/sizeof(terminalChars[0]) - 1);
static const nsReadEndCondition theEndCondition(terminalChars);
nsReadingIterator<PRUnichar> start, end;
aScanner.CurrentPosition(start);
aScanner.EndReading(end);
nsresult result=aScanner.ReadUntil(start, end, terminals,PR_FALSE);
nsresult result=aScanner.ReadUntil(start, end, theEndCondition, PR_FALSE);
if (NS_SUCCEEDED(result)) {
PRUnichar ch;

240
mozilla/parser/htmlparser/src/nsScanner.cpp
View File

@@ -48,6 +48,25 @@ nsScannerString::ReplaceCharacter(nsReadingIterator<PRUnichar>& aPosition,
*pos = aChar;
}
nsReadEndCondition::nsReadEndCondition(const PRUnichar* aTerminateChars) :
mChars(aTerminateChars), mFilter(PRUnichar(~0)) // All bits set
{
// Build filter that will be used to filter out characters with
// bits that none of the terminal chars have. This works very well
// because terminal chars often have only the last 4-6 bits set and
// normal ascii letters have bit 7 set. Other letters have even higher
// bits set.
// Calculate filter
const PRUnichar *current = aTerminateChars;
PRUnichar terminalChar = *current;
while (terminalChar) {
mFilter &= ~terminalChar;
++current;
terminalChar = *current;
}
}
static NS_DEFINE_CID(kCharsetConverterManagerCID, NS_ICHARSETCONVERTERMANAGER_CID);
const char* kBadHTMLText="<H3>Oops...</H3>You just tried to read a non-existent document: <BR>";
@@ -533,17 +552,18 @@ nsresult nsScanner::SkipWhitespace(void) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> current, end;
PRBool found=PR_FALSE;
nsReadingIterator<PRUnichar> current;
PRBool found;
PRBool skipped = PR_FALSE;
mNewlinesSkipped = 0;
current = mCurrentPosition;
end = mEndPosition;
while(current != end) {
theChar=*current;
PRUnichar theChar=0;
nsresult result=Peek(theChar);
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
switch(theChar) {
case '\n': mNewlinesSkipped++;
case ' ' :
@@ -559,19 +579,22 @@ nsresult nsScanner::SkipWhitespace(void) {
if(!found) {
break;
}
else {
++current;
}
++current;
theChar = *current;
skipped = PR_TRUE;
}
SetPosition(current);
if (current == end) {
if (!skipped)
return NS_OK;
if (current == mEndPosition) {
SetPosition(current);
return Eof();
}
//DoErrTest(aString);
SetPosition(current);
return result;
return NS_OK;
}
@@ -1156,156 +1179,113 @@ nsresult nsScanner::ReadWhile(nsString& aString,
* @return error code
*/
nsresult nsScanner::ReadUntil(nsAWritableString& aString,
const nsAFlatString& aTerminalSet,
const nsReadEndCondition& aEndCondition,
PRBool addTerminal)
{
if (!mSlidingBuffer) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
const PRUnichar* setstart = aTerminalSet.get();
nsReadingIterator<PRUnichar> origin, current;
const PRUnichar* setstart = aEndCondition.mChars;
const PRUnichar* setcurrent;
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
setcurrent = setstart;
theChar=*current;
if(theChar) {
while (*setcurrent) {
if (*setcurrent == theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
goto found;
}
++setcurrent;
}
}
++current;
}
found:
SetPosition(current);
if (current == end) {
AppendUnicodeTo(origin, current, aString);
return Eof();
}
//DoErrTest(aString);
return result;
}
/**
* Consume characters until you encounter one contained in given
* input set.
*
* @update gess 3/25/98
* @param aString will contain the result of this method
* @param aTerminalSet is an ordered string that contains
* the set of INVALID characters
* @return error code
*/
nsresult nsScanner::ReadUntil(nsAWritableString& aString,
const nsAFlatCString& aTerminalSet,
PRBool addTerminal)
{
if (!mSlidingBuffer) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
const char* setstart = aTerminalSet.get();
const char* setcurrent;
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
// Filter out completely wrong characters
// Check if all bits are in the required area
if(!(theChar & aEndCondition.mFilter)) {
// They were. Do a thorough check.
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
setcurrent = setstart;
theChar=*current;
if(theChar) {
setcurrent = setstart;
while (*setcurrent) {
if (*setcurrent == theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
goto found;
}
++setcurrent;
}
}
++current;
theChar = *current;
}
found:
// If we are here, we didn't find any terminator in the string and
// current = mEndPosition
SetPosition(current);
if (current == end) {
AppendUnicodeTo(origin, current, aString);
return Eof();
}
AppendUnicodeTo(origin, current, aString);
return Eof();
found:
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
SetPosition(current);
//DoErrTest(aString);
return result;
return NS_OK;
}
nsresult nsScanner::ReadUntil(nsReadingIterator<PRUnichar>& aStart,
nsReadingIterator<PRUnichar>& aEnd,
const nsAFlatString& aTerminalSet,
const nsReadEndCondition &aEndCondition,
PRBool addTerminal)
{
if (!mSlidingBuffer) {
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
const PRUnichar* setstart = aTerminalSet.get();
nsReadingIterator<PRUnichar> origin, current;
const PRUnichar* setstart = aEndCondition.mChars;
const PRUnichar* setcurrent;
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
setcurrent = setstart;
theChar=*current;
if(theChar) {
PRUnichar theChar=0;
nsresult result=Peek(theChar);
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
// Filter out completely wrong characters
// Check if all bits are in the required area
if(!(theChar & aEndCondition.mFilter)) {
// They were. Do a thorough check.
setcurrent = setstart;
while (*setcurrent) {
if (*setcurrent == theChar) {
if(addTerminal)
++current;
aStart = origin;
aEnd = current;
goto found;
}
++setcurrent;
++setcurrent;
}
}
++current;
theChar = *current;
}
found:
// If we are here, we didn't find any terminator in the string and
// current = mEndPosition
SetPosition(current);
if (current == end) {
aStart = origin;
aEnd = current;
return Eof();
}
aStart = origin;
aEnd = current;
return Eof();
return result;
found:
if(addTerminal)
++current;
aStart = origin;
aEnd = current;
SetPosition(current);
return NS_OK;
}
/**
@@ -1323,36 +1303,32 @@ nsresult nsScanner::ReadUntil(nsAWritableString& aString,
return kEOF;
}
PRUnichar theChar=0;
nsresult result=Peek(theChar);
nsReadingIterator<PRUnichar> origin, current, end;
nsReadingIterator<PRUnichar> origin, current;
origin = mCurrentPosition;
current = origin;
end = mEndPosition;
while(current != end) {
theChar=*current;
if(theChar) {
if(aTerminalChar==theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
break;
}
PRUnichar theChar;
nsresult result=Peek(theChar);
NS_ENSURE_SUCCESS(result, result);
while (current != mEndPosition) {
if (aTerminalChar == theChar) {
if(addTerminal)
++current;
AppendUnicodeTo(origin, current, aString);
SetPosition(current);
return NS_OK;
}
++current;
theChar = *current;
}
// If we are here, we didn't find any terminator in the string and
// current = mEndPosition
AppendUnicodeTo(origin, current, aString);
SetPosition(current);
if (current == end) {
AppendUnicodeTo(origin, current, aString);
return Eof();
}
//DoErrTest(aString);
return result;
return Eof();
}

27
mozilla/parser/htmlparser/src/nsScanner.h
View File

@@ -54,6 +54,16 @@ class nsScannerString : public nsSlidingString {
PRUnichar aChar);
};
class nsReadEndCondition {
public:
const PRUnichar *mChars;
PRUnichar mFilter;
explicit nsReadEndCondition(const PRUnichar* aTerminateChars);
private:
nsReadEndCondition(const nsReadEndCondition& aOther); // No copying
void operator=(const nsReadEndCondition& aOther); // No assigning
};
class nsScanner {
public:
@@ -216,25 +226,12 @@ class nsScanner {
* @return error code
*/
nsresult ReadUntil(nsAWritableString& aString,
const nsAFlatString& aTermSet,
const nsReadEndCondition& aEndCondition,
PRBool addTerminal);
nsresult ReadUntil(nsAWritableString& aString,
const nsAFlatCString& aTermSet,
PRBool addTerminal);
nsresult ReadUntil(nsAWritableString& aString,
const char* aTerminalSet,
PRBool addTerminal)
{
return ReadUntil(aString,
nsDependentCString(aTerminalSet),
addTerminal);
}
nsresult ReadUntil(nsReadingIterator<PRUnichar>& aStart,
nsReadingIterator<PRUnichar>& aEnd,
const nsAFlatString& aTerminalSet,
const nsReadEndCondition& aEndCondition,
PRBool addTerminal);