/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
 *
 * ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is the JavaScript 2 Prototype.
 *
 * The Initial Developer of the Original Code is
 * Netscape Communications Corporation.
 * Portions created by the Initial Developer are Copyright (C) 1998
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

#include <vector>

#include "systemtypes.h"
#include "js2value.h"

#include "numerics.h"
#include "lexer.h"

namespace JS = JavaScript;


// Create a new Lexer for lexing the provided source code.  The Lexer will
// intern identifiers, keywords, and regular expressions in the designated
// world.
JS::Lexer::Lexer(World &world, const String &source, const String &sourceLocation, uint32 initialLineNum):
        world(world), reader(source, sourceLocation, initialLineNum)
{
    nextToken = tokens;
    nTokensFwd = 0;
#ifdef DEBUG
    nTokensBack = 0;
#endif
    lexingUnit = false;
}


// Skip past the next token, which must have been either peeked or read and then unread.
// skip is faster than get but must not be called if the next token has not been seen yet.
void JS::Lexer::skip()
{
    ASSERT(nTokensFwd);
    if (++nextToken == tokens + tokenBufferSize)
        nextToken = tokens;
    --nTokensFwd;
    DEBUG_ONLY(++nTokensBack);
}


// Get and return the next token.  The token remains valid until the next
// call to this Lexer.  If the Reader reached the end of file, return a
// Token whose Kind is end.  The caller may alter the value of this Token
// (in particular, take control over the auto_ptr's data), but if it does so,
// the caller is not allowed to unget this Token.
//
// If preferRegExp is true, a / will be preferentially interpreted as
// starting a regular expression; otherwise, a / will be preferentially
// interpreted as division or /=.
const JS::Token &JS::Lexer::get(bool preferRegExp)
{
    const Token &t = peek(preferRegExp);
    if (++nextToken == tokens + tokenBufferSize)
        nextToken = tokens;
    --nTokensFwd;
    DEBUG_ONLY(++nTokensBack);
    return t;
}


// Peek at the next token using the given preferRegExp setting.  If that
// token's kind matches the given kind, consume that token and return it.
// Otherwise, do not consume that token and return nil.
const JS::Token *JS::Lexer::eat(bool preferRegExp, Token::Kind kind)
{
    const Token &t = peek(preferRegExp);
    if (t.kind != kind)
        return 0;
    if (++nextToken == tokens + tokenBufferSize)
        nextToken = tokens;
    --nTokensFwd;
    DEBUG_ONLY(++nTokensBack);
    return &t;
}


// Return the next token without consuming it.
//
// If preferRegExp is true, a / will be preferentially interpreted as
// starting a regular expression; otherwise, a / will be preferentially
// interpreted as division or /=. A subsequent call to peek or get will
// return the same token; that call must be presented with the same value
// for preferRegExp.
const JS::Token &JS::Lexer::peek(bool preferRegExp)
{
    // Use an already looked-up token if there is one.
    if (nTokensFwd) {
        ASSERT(savedPreferRegExp[nextToken - tokens] == preferRegExp);
    } else {
        lexToken(preferRegExp);
        nTokensFwd = 1;
#ifdef DEBUG
        savedPreferRegExp[nextToken - tokens] = preferRegExp;
        if (nTokensBack == tokenLookahead) {
            nTokensBack = tokenLookahead-1;
            if (tokenGuard)
                (nextToken >= tokens+tokenLookahead ?
                 nextToken-tokenLookahead :
                 nextToken+tokenBufferSize-tokenLookahead)->valid = false;
        }
#endif
    }
    return *nextToken;
}


#ifdef DEBUG
// Change the setting of preferRegExp for an already peeked token.
// The token must not be one for which that setting mattered.
//
// THIS IS A DANGEROUS FUNCTION!
// Use it only if you can be prove that the already peeked token does not
// start with a slash.
void JS::Lexer::redesignate(bool preferRegExp)
{
    if (nTokensFwd) {
        ASSERT(savedPreferRegExp[nextToken - tokens] != preferRegExp);
        ASSERT(!(nextToken->hasKind(Token::regExp) ||
                 nextToken->hasKind(Token::divide) ||
                 nextToken->hasKind(Token::divideEquals)));
        savedPreferRegExp[nextToken - tokens] = preferRegExp;
    }
}
#endif


// Unread the last token.  This call may be called to unread at most
// tokenBufferSize tokens at a time (where a peek also counts as temporarily
// reading and unreading one token). When a token that has been unread is
// peeked or read again, the same value must be passed in preferRegExp as for
// the first time that token was read or peeked.
void JS::Lexer::unget()
{
    ASSERT(nTokensBack--);
    nTokensFwd++;
    if (nextToken == tokens)
        nextToken = tokens + tokenBufferSize;
    --nextToken;
}


// Report a syntax error at the backUp-th last character read by the Reader.
// In other words, if backUp is 0, the error is at the next character to be
// read by the Reader; if backUp is 1, the error is at the last character
// read by the Reader, and so forth.
void JS::Lexer::syntaxError(const char *message, uint backUp)
{
    reader.unget(backUp);
    reader.error(Exception::syntaxError, widenCString(message), reader.getPos());
}


// Get the next character from the reader, skipping any Unicode format-control
// (Cf) characters.
inline char16 JS::Lexer::getChar()
{
    char16 ch = reader.get();
    if (char16Value(ch) >= firstFormatChar)
        ch = internalGetChar(ch);
    return ch;
}


// Helper for getChar()
char16 JS::Lexer::internalGetChar(char16 ch)
{
    while (isFormat(ch))
        ch = reader.get();
    return ch;
}


// Peek the next character from the reader, skipping any Unicode
// format-control (Cf) characters, which are read and discarded.
inline char16 JS::Lexer::peekChar()
{
    char16 ch = reader.peek();
    if (char16Value(ch) >= firstFormatChar)
        ch = internalPeekChar(ch);
    return ch;
}


// Helper for peekChar()
char16 JS::Lexer::internalPeekChar(char16 ch)
{
    while (isFormat(ch)) {
        reader.get();
        ch = reader.peek();
    }
    return ch;
}


// Peek the next character from the reader, skipping any Unicode
// format-control (Cf) characters, which are read and discarded.  If the
// peeked character matches ch, read that character and return true;
// otherwise return false.  ch must not be null.
bool JS::Lexer::testChar(char16 ch)
{
    ASSERT(ch);     // If ch were null, it could match the eof null.
    char16 ch2 = peekChar();
    if (ch == ch2) {
        reader.get();
        return true;
    }
    return false;
}


// A backslash has been read.  Read the rest of the escape code.
// Return the interpreted escaped character.  Throw an exception if the
// escape is not valid. If unicodeOnly is true, allow only \uxxxx escapes.
char16 JS::Lexer::lexEscape(bool unicodeOnly)
{
    char16 ch = getChar();
    int nDigits;

    if (!unicodeOnly || ch == 'u')
        switch (ch) {
          case '0':
            // Make sure that the next character isn't a digit.
            ch = peekChar();
            if (!isASCIIDecimalDigit(ch))
                return 0x00;
/*
            // Point to the next character in the error message
            getChar();
            break;
*/
			/* E3 compatibility, parse the sequence as octal */
            {
                uint32 n = 0;
                while (isASCIIDecimalDigit(ch)) {
                    ch = getChar();
                    n = (n << 3) | (ch - '0');
					ch = peekChar();
                }
                return static_cast<char16>(n);
            }


          case 'b':
            return 0x08;
          case 'f':
            return 0x0C;
          case 'n':
            return 0x0A;
          case 'r':
            return 0x0D;
          case 't':
            return 0x09;
          case 'v':
            return 0x0B;

          case 'x':
            nDigits = 2;
            goto lexHex;
          case 'u':
            nDigits = 4;
          lexHex:
            {
                uint32 n = 0;
                while (nDigits--) {
                    ch = getChar();
                    uint digit;
                    if (!isASCIIHexDigit(ch, digit)) {
						/* E3 compatibility, back off */
						// goto error;
						do {
							reader.unget();
							ch = peekChar();
						} while (ch != '\\');
                        return getChar();
					}
                    n = (n << 4) | digit;
                }
                return static_cast<char16>(n);
            }
          default:
/*
            if (!reader.getEof(ch)) {
                CharInfo chi(ch);
                if (!isAlphanumeric(chi) && !isLineBreak(chi))
                    return ch;
            }
*/
              return ch;
        }
//  error:
    syntaxError("Bad escape code");
    return 0;
}


// Read an identifier into s.  The initial value of s is ignored and cleared.
// Return true if an escape code has been encountered.
// If allowLeadingDigit is true, allow the first character of s to be a digit,
// just like any continuing identifier character.
bool JS::Lexer::lexIdentifier(String &s, bool allowLeadingDigit)
{
    reader.beginRecording(s);
    bool hasEscape = false;

    while (true) {
        char16 ch = getChar();
        char16 ch2 = ch;
        if (ch == '\\') {
            ch2 = lexEscape(true);
            hasEscape = true;
        }
        CharInfo chi2(ch2);

        if (!(allowLeadingDigit ? isIdContinuing(chi2) :
              isIdLeading(chi2))) {
            if (ch == '\\')
                syntaxError("Identifier escape expands into non-identifier character");
            else
                reader.unget();
            break;
        }
        reader.recordChar(ch2);
        allowLeadingDigit = true;
    }
    reader.endRecording();
    return hasEscape;
}


// Read a numeric literal into nextToken->chars and nextToken->value.
// Return true if the numeric literal is followed by a unit, but don't read
// the unit yet.
bool JS::Lexer::lexNumeral()
{
    int hasDecimalPoint = 0;
    bool hexadecimal = false;
    bool octal = false;
    String &s = nextToken->chars;
    uint digit;

    reader.beginRecording(s);
    char16 ch = getChar();
    if (ch == '0') {
        reader.recordChar('0');
        ch = getChar();
        if ((ch&~0x20) == 'X') {
            size_t pos = reader.getPos();
            char16 ch2 = getChar();
            if (isASCIIHexDigit(ch2, digit)) {
                hexadecimal = true;
                reader.recordChar(ch);
                do {
                    reader.recordChar(ch2);
                    ch2 = getChar();
                } while (isASCIIHexDigit(ch2, digit));
                ch = ch2;
            } else
                reader.setPos(pos);
            goto done;
        } else if (isASCIIOctalDigit(ch)) {
            // Backward compatible hack, support octal for SpiderMonkey's sake
            octal = true;
            while (isASCIIOctalDigit(ch)) {
                reader.recordChar(ch);
                ch = getChar();
            }
            goto done;
//          syntaxError("Numeric constant syntax error");
        }
    }
    while (isASCIIDecimalDigit(ch) || ch == '.' && !hasDecimalPoint++) {
        reader.recordChar(ch);
        ch = getChar();
    }
    if ((ch&~0x20) == 'E') {
        size_t pos = reader.getPos();
        char16 ch2 = getChar();
        char16 sign = 0;
        if (ch2 == '+' || ch2 == '-') {
            sign = ch2;
            ch2 = getChar();
        }
        if (isASCIIDecimalDigit(ch2)) {
            reader.recordChar(ch);
            if (sign)
                reader.recordChar(sign);
            do {
                reader.recordChar(ch2);
                ch2 = getChar();
            } while (isASCIIDecimalDigit(ch2));
            ch = ch2;
        } else
            reader.setPos(pos);
    }

  done:
    // At this point the reader is just past the character ch, which
    // is the first non-formatting character that is not part of the
    // number.
    reader.endRecording();
    const char16 *sBegin = s.data();
    const char16 *sEnd = sBegin + s.size();
    const char16 *numEnd;
    nextToken->value = hexadecimal ? 
                            stringToInteger(sBegin, sEnd, numEnd, 16)
                          : octal ? 
                                stringToInteger(sBegin, sEnd, numEnd, 8)
                              : stringToDouble(sBegin, sEnd, numEnd);
    ASSERT(numEnd == sEnd);
    reader.unget();
    ASSERT(ch == reader.peek());
    return isIdContinuing(ch) || ch == '\\';
}


// Read a string literal into s.  The initial value of s is ignored and
// cleared. The opening quote has already been read into separator.
void JS::Lexer::lexString(String &s, char16 separator)
{
    char16 ch;

    reader.beginRecording(s);
    while ((ch = reader.get()) != separator) {
        CharInfo chi(ch);
        if (!isFormat(chi)) {
            if (ch == '\\')
                ch = lexEscape(false);
            else if (reader.getEof(ch) || isLineBreak(chi))
                syntaxError("Unterminated string literal");
            reader.recordChar(ch);
        }
    }
    reader.endRecording();
}


// Read a regular expression literal.  Store the regular expression in
// nextToken->id and the flags in nextToken->chars.
// The opening slash has already been read.
void JS::Lexer::lexRegExp()
{
    String s;
    char16 prevCh = 0;

    reader.beginRecording(s);
    while (true) {
        char16 ch = getChar();
        CharInfo chi(ch);
        if (reader.getEof(ch) || isLineBreak(chi))
            syntaxError("Unterminated regular expression literal");
        if (prevCh == '\\') {
            reader.recordChar(ch);
            // Ignore slashes and backslashes immediately after a backslash
            prevCh = 0;
        } else if (ch != '/') {
            reader.recordChar(ch);
            prevCh = ch;
        } else
            break;
    }
    reader.endRecording();
    nextToken->id = &world.identifiers[s];

    lexIdentifier(nextToken->chars, true);
}


// Read a token from the Reader and store it at *nextToken.
// If the Reader reached the end of file, store a Token whose Kind is end.
void JS::Lexer::lexToken(bool preferRegExp)
{
    Token &t = *nextToken;
    t.lineBreak = false;
    t.id = 0;
    // Don't really need to waste time clearing this string here
    //clear(t.chars);
    Token::Kind kind;

    if (lexingUnit) {
        if (reader.peek() == '_')
            syntaxError("Unit suffix may not begin with an underscore", 0);
        lexIdentifier(t.chars, false);
        ASSERT(t.chars.size());
        kind = Token::unit;   // unit
        lexingUnit = false;
    } else {
      next:
        char16 ch = reader.get();
        if (reader.getEof(ch)) {
          endOfInput:
            t.pos = reader.getPos() - 1;
            kind = Token::end;
        } else {
            char16 ch2;
            CharInfo chi(ch);

            switch (cGroup(chi)) {
              case CharInfo::FormatGroup:
              case CharInfo::WhiteGroup:
                goto next;

              case CharInfo::IdGroup:
                t.pos = reader.getPos() - 1;
              readIdentifier:
                {
                    reader.unget();
                    String s;
                    bool hasEscape = lexIdentifier(s, false);
                    t.id = &world.identifiers[s];
                    kind = hasEscape ? Token::identifier : t.id->tokenKind;
                }
                break;

              case CharInfo::NonIdGroup:
              case CharInfo::IdContinueGroup:
                t.pos = reader.getPos() - 1;
                switch (ch) {
                  case '(':
                    kind = Token::openParenthesis;  // (
                    break;
                  case ')':
                    kind = Token::closeParenthesis; // )
                    break;
                  case '[':
                    kind = Token::openBracket;      // [
                    break;
                  case ']':
                    kind = Token::closeBracket;     // ]
                    break;
                  case '{':
                    kind = Token::openBrace;        // {
                    break;
                  case '}':
                    kind = Token::closeBrace;       // }
                    break;
                  case ',':
                    kind = Token::comma;            // ,
                    break;
                  case ';':
                    kind = Token::semicolon;        // ;
                    break;
                  case '.':
                    kind = Token::dot;              // .
                    ch2 = getChar();
                    if (isASCIIDecimalDigit(ch2)) {
                        reader.setPos(t.pos);
                        goto number;                // decimal point
                    } else if (ch2 == '.') {
                        kind = Token::doubleDot;    // ..
                        if (testChar('.'))
                            kind = Token::tripleDot;// ...
                    } else
                        reader.unget();
                    break;
                  case ':':
                    kind = Token::colon;            // :
                    if (testChar(':'))
                        kind = Token::doubleColon;  // ::
                    break;
                  case '#':
                    kind = Token::pound;            // #
                    break;
                  case '@':
                    kind = Token::at;               // @
                    break;
                  case '?':
                    kind = Token::question;         // ?
                    break;

                  case '~':
                    kind = Token::complement;       // ~
                    break;
                  case '!':
                    kind = Token::logicalNot;       // !
                    if (testChar('=')) {
                        kind = Token::notEqual;     // !=
                        if (testChar('='))
                            kind = Token::notIdentical; // !==
                    }
                    break;

                  case '*':
                    kind = Token::times;            // * *=
                  tryAssignment:
                    if (testChar('='))
                        kind = Token::Kind(kind + Token::timesEquals - Token::times);
                    break;

                  case '/':
                    kind = Token::divide;           // /
                    ch = getChar();
                    if (ch == '/') {                // // comment
                        do {
                            ch = reader.get();
                            if (reader.getEof(ch))
                                goto endOfInput;
                        } while (!isLineBreak(ch));
                        goto endOfLine;
                    } else if (ch == '*') {         // /*comment*/
                        ch = 0;
                        do {
                            ch2 = ch;
                            ch = getChar();
                            if (isLineBreak(ch)) {
                                reader.beginLine();
                                t.lineBreak = true;
                            } else if (reader.getEof(ch))
                                syntaxError("Unterminated /* comment");
                        } while (ch != '/' || ch2 != '*');
                        goto next;
                    } else {
                        reader.unget();
                        if (preferRegExp) {         // Regular expression
                            kind = Token::regExp;
                            lexRegExp();
                        } else
                            goto tryAssignment;     // /=
                    }
                    break;

                  case '%':
                    kind = Token::modulo;           // %
                    goto tryAssignment;             // %=

                  case '+':
                    kind = Token::plus;             // +
                    if (testChar('+'))
                        kind = Token::increment;    // ++
                    else
                        goto tryAssignment;         // +=
                    break;

                  case '-':
                    kind = Token::minus;            // -
                    ch = getChar();
                    if (ch == '-')
                        kind = Token::decrement;    // --
                    else if (ch == '>')
                        kind = Token::arrow;        // ->
                    else {
                        reader.unget();
                        goto tryAssignment;         // -=
                    }
                    break;

                  case '&':
                    kind = Token::bitwiseAnd;       // & && &= &&=
                  logical:
                    if (testChar(ch))
                        kind = Token::Kind(kind - Token::bitwiseAnd + Token::logicalAnd);
                    goto tryAssignment;
                  case '^':
                    kind = Token::bitwiseXor;       // ^ ^^ ^= ^^=
                    goto logical;
                  case '|':
                    kind = Token::bitwiseOr;        // | || |= ||=
                    goto logical;

                  case '=':
                    kind = Token::assignment;       // =
                    if (testChar('=')) {
                        kind = Token::equal;        // ==
                        if (testChar('='))
                            kind = Token::identical; // ===
                    }
                    break;

                  case '<':
                    kind = Token::lessThan;         // <
                    if (testChar('<')) {
                        kind = Token::leftShift;    // <<
                        goto tryAssignment;         // <<=
                    }
                  comparison:
                    if (testChar('='))              // <= >=
                        kind = Token::Kind(kind + Token::lessThanOrEqual - Token::lessThan);
                    break;
                  case '>':
                    kind = Token::greaterThan;      // >
                    if (testChar('>')) {
                        kind = Token::rightShift;   // >>
                        if (testChar('>'))
                            kind = Token::logicalRightShift; // >>>
                        goto tryAssignment;         // >>= >>>=
                    }
                    goto comparison;

                  case '\\':
                    goto readIdentifier;            // An identifier that starts with an escape

                  case '\'':
                  case '"':
                    kind = Token::string;           // 'string' "string"
                    lexString(t.chars, ch);
                    break;

                  case '0':
                  case '1':
                  case '2':
                  case '3':
                  case '4':
                  case '5':
                  case '6':
                  case '7':
                  case '8':
                  case '9':
                    reader.unget();                 // Number
                  number:
                    kind = Token::number;
#ifdef PARSE_UNIT
                    lexingUnit = 
#else
						lexNumeral();
#endif
                    break;

                  default:
                    syntaxError("Bad character");
                }
                break;

              case CharInfo::LineBreakGroup:
              endOfLine:
                reader.beginLine();
                t.lineBreak = true;
                goto next;
            }
        }
    }
    t.kind = kind;
#ifdef DEBUG
    t.valid = true;
#endif
}