Ignore:
Timestamp:
Nov 7, 2011, 9:54:15 AM (14 years ago)
Author:
[email protected]
Message:

Towards 8 Bit Strings: Templatize JSC::Lexer class by character type
​https://bugs.webkit.org/show_bug.cgi?id=71331

Source/JavaScriptCore:

Change the Lexer class to be a template class based on the character
type of the source. In the process updated the parseIdentifier()
and parseString() methods to create 8 bit strings where possible.
Also added some helper methods for accumulating temporary string
data in the 8 and 16 bit vectors.

Changed the SourceProvider::data() virtual method to return a
StringImpl* instead of a UChar*.

Updated the KeywordLookup generator to create code to match keywords
for both 8 and 16 bit source strings.

Due to a compiler bug (<rdar://problem/10194295>) moved enum
definition outside of Lexer class declaration. Remove second enum
no longer needed.

Reviewed by Darin Adler.

  • KeywordLookupGenerator.py:
  • interpreter/Interpreter.cpp:

(JSC::Interpreter::callEval):

  • parser/Lexer.cpp:

(JSC::::Lexer):
(JSC::::~Lexer):
(JSC::::getInvalidCharMessage):
(JSC::::currentCharacter):
(JSC::::setCode):
(JSC::::internalShift):
(JSC::::shift):
(JSC::::peek):
(JSC::::getUnicodeCharacter):
(JSC::::shiftLineTerminator):
(JSC::::lastTokenWasRestrKeyword):
(JSC::::record8):
(JSC::::append8):
(JSC::::append16):
(JSC::::record16):
(JSC::::parseIdentifier):
(JSC::::parseIdentifierSlowCase):
(JSC::::parseString):
(JSC::::parseStringSlowCase):
(JSC::::parseHex):
(JSC::::parseOctal):
(JSC::::parseDecimal):
(JSC::::parseNumberAfterDecimalPoint):
(JSC::::parseNumberAfterExponentIndicator):
(JSC::::parseMultilineComment):
(JSC::::nextTokenIsColon):
(JSC::::lex):
(JSC::::scanRegExp):
(JSC::::skipRegExp):
(JSC::::clear):
(JSC::::sourceCode):

  • parser/Lexer.h:

(JSC::Lexer::append16):
(JSC::Lexer::currentOffset):
(JSC::Lexer::setOffsetFromCharOffset):
(JSC::::isWhiteSpace):
(JSC::::isLineTerminator):
(JSC::::convertHex):
(JSC::::convertUnicode):
(JSC::::makeIdentifier):
(JSC::::setCodeStart):
(JSC::::makeIdentifierLCharFromUChar):
(JSC::::lexExpectIdentifier):

  • parser/Parser.cpp:

(JSC::Parser::Parser):
(JSC::Parser::parseProperty):
(JSC::Parser::parseMemberExpression):

  • parser/Parser.h:

(JSC::Parser::next):
(JSC::Parser::nextExpectIdentifier):

  • parser/ParserArena.h:

(JSC::IdentifierArena::makeIdentifier):
(JSC::IdentifierArena::makeIdentifierLCharFromUChar):

  • parser/SourceCode.h:

(JSC::SourceCode::subExpression):

  • parser/SourceProvider.h:

(JSC::UStringSourceProvider::stringData):

  • parser/SourceProviderCache.h:
  • parser/SyntaxChecker.h:
  • runtime/FunctionPrototype.cpp:

(JSC::insertSemicolonIfNeeded):

  • runtime/Identifier.cpp:

(JSC::IdentifierTable::add):
(JSC::IdentifierLCharFromUCharTranslator::hash):
(JSC::IdentifierLCharFromUCharTranslator::equal):
(JSC::IdentifierLCharFromUCharTranslator::translate):
(JSC::Identifier::add8):

  • runtime/Identifier.h:

(JSC::Identifier::Identifier):
(JSC::Identifier::createLCharFromUChar):
(JSC::Identifier::canUseSingleCharacterString):
(JSC::IdentifierCharBufferTranslator::hash):
(JSC::IdentifierCharBufferTranslator::equal):
(JSC::IdentifierCharBufferTranslator::translate):
(JSC::Identifier::add):
(JSC::Identifier::equal):
(JSC::IdentifierTable::add):

  • runtime/JSGlobalObjectFunctions.cpp:

(JSC::decode):
(JSC::parseIntOverflow):
(JSC::globalFuncUnescape):

  • runtime/JSGlobalObjectFunctions.h:

(JSC::parseIntOverflow):

  • runtime/LiteralParser.cpp:

(JSC::LiteralParser::tryJSONPParse):
(JSC::LiteralParser::Lexer::lexString):

  • wtf/text/StringImpl.h:

Source/WebCore:

Changed the SourceProvider::data() virtual method to return a
StringImpl* instead of a UChar*.
Changed Identifier() constructor to use JSGlobalData*.

Reviewed by Darin Adler.

No new tests - refactored SourceProvider class and sub-classes.

  • bindings/js/CachedScriptSourceProvider.h:

(WebCore::CachedScriptSourceProvider::stringData):

  • bindings/js/StringSourceProvider.h:

(WebCore::StringSourceProvider::stringData):

  • bridge/qt/qt_runtime.cpp:

(JSC::Bindings::convertQVariantToValue):

Source/WebKit/qt:

Changed Identifier() constructor to use JSGlobalData*.

Reviewed by Darin Adler.

  • Api/qwebframe.cpp:

(QWebFrame::addToJavaScriptWindowObject):

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/JavaScriptCore/parser/Lexer.cpp

    r98887 r99436  
    227227};
    228228
    229 Lexer::Lexer(JSGlobalData* globalData)
     229template <typename T>
     230Lexer<T>::Lexer(JSGlobalData* globalData)
    230231    : m_isReparsing(false)
    231232    , m_globalData(globalData)
    … …  
    233234}
    234235
    235 Lexer::~Lexer()
    236 {
    237 }
    238    
    239 UString Lexer::getInvalidCharMessage()
     236template <typename T>
     237Lexer<T>::~Lexer()
     238{
     239}
     240
     241template <typename T>
     242UString Lexer<T>::getInvalidCharMessage()
    240243{
    241244    switch (m_current) {
    … …  
    259262}
    260263
    261 ALWAYS_INLINE const UChar* Lexer::currentCharacter() const
     264template <typename T>
     265ALWAYS_INLINE const T* Lexer<T>::currentCharacter() const
    262266{
    263267    ASSERT(m_code <= m_codeEnd);
    … …  
    265269}
    266270
    267 ALWAYS_INLINE int Lexer::currentOffset() const
    268 {
    269     return currentCharacter() - m_codeStart;
    270 }
    271 
    272 void Lexer::setCode(const SourceCode& source, ParserArena* arena)
     271template <typename T>
     272void Lexer<T>::setCode(const SourceCode& source, ParserArena* arena)
    273273{
    274274    m_arena = &arena->identifierArena();
    275 
     275   
    276276    m_lineNumber = source.firstLine();
    277277    m_delimited = false;
    278278    m_lastToken = -1;
    279 
    280     const UChar* data = source.provider()->data();
     279   
     280    const StringImpl* sourceString = source.provider()->stringData();
     281
     282    if (sourceString)
     283        setCodeStart(sourceString);
     284    else
     285        m_codeStart = 0;
    281286
    282287    m_source = &source;
    283     m_codeStart = data;
    284     m_code = data + source.startOffset();
    285     m_codeEnd = data + source.endOffset();
     288    m_code = m_codeStart + source.startOffset();
     289    m_codeEnd = m_codeStart + source.endOffset();
    286290    m_error = false;
    287291    m_atLineStart = true;
    288292    m_lexErrorMessage = UString();
    289 
     293   
    290294    m_buffer8.reserveInitialCapacity(initialReadBufferCapacity);
    291295    m_buffer16.reserveInitialCapacity((m_codeEnd - m_code) / 2);
    292 
     296   
    293297    if (LIKELY(m_code < m_codeEnd))
    294298        m_current = *m_code;
    … …  
    298302}
    299303
    300 template <int shiftAmount, Lexer::ShiftType shouldBoundsCheck> ALWAYS_INLINE void Lexer::internalShift()
    301 {
    302     if (shouldBoundsCheck == DoBoundsCheck) {
    303         // Faster than an if-else sequence
    304         ASSERT(m_current != -1);
    305         m_current = -1;
    306         m_code += shiftAmount;
    307         if (LIKELY(m_code < m_codeEnd))
    308             m_current = *m_code;
    309     } else {
    310         m_code += shiftAmount;
     304template <typename T>
     305template <int shiftAmount> ALWAYS_INLINE void Lexer<T>::internalShift()
     306{
     307    m_code += shiftAmount;
     308    m_current = *m_code;
     309}
     310
     311template <typename T>
     312ALWAYS_INLINE void Lexer<T>::shift()
     313{
     314    // Faster than an if-else sequence
     315    ASSERT(m_current != -1);
     316    m_current = -1;
     317    m_code++;
     318    if (LIKELY(m_code < m_codeEnd))
    311319        m_current = *m_code;
    312     }
    313 }
    314 
    315 ALWAYS_INLINE void Lexer::shift()
    316 {
    317     internalShift<1, DoBoundsCheck>();
    318 }
    319 
    320 ALWAYS_INLINE int Lexer::peek(int offset)
     320}
     321
     322template <typename T>
     323ALWAYS_INLINE int Lexer<T>::peek(int offset)
    321324{
    322325    // Only use if necessary
    323326    ASSERT(offset > 0 && offset < 5);
    324     const UChar* code = m_code + offset;
     327    const T* code = m_code + offset;
    325328    return (code < m_codeEnd) ? *code : -1;
    326329}
    327330
    328 int Lexer::getUnicodeCharacter()
     331template <typename T>
     332int Lexer<T>::getUnicodeCharacter()
    329333{
    330334    int char1 = peek(1);
    … …  
    343347}
    344348
    345 void Lexer::shiftLineTerminator()
     349template <typename T>
     350void Lexer<T>::shiftLineTerminator()
    346351{
    347352    ASSERT(isLineTerminator(m_current));
    … …  
    357362}
    358363
    359 ALWAYS_INLINE bool Lexer::lastTokenWasRestrKeyword() const
     364template <typename T>
     365ALWAYS_INLINE bool Lexer<T>::lastTokenWasRestrKeyword() const
    360366{
    361367    return m_lastToken == CONTINUE || m_lastToken == BREAK || m_lastToken == RETURN || m_lastToken == THROW;
    … …  
    412418}
    413419
    414 inline void Lexer::record8(int c)
     420template <typename T>
     421inline void Lexer<T>::record8(int c)
    415422{
    416423    ASSERT(c >= 0);
    417424    ASSERT(c <= 0xFF);
    418     m_buffer8.append(static_cast<char>(c));
    419 }
    420 
    421 inline void Lexer::record16(UChar c)
     425    m_buffer8.append(static_cast<LChar>(c));
     426}
     427
     428template <typename T>
     429inline void Lexer<T>::append8(const T* p, size_t length)
     430{
     431    // FIXME: Change three occurrances of m_buffer16 to m_buffer8 and
     432    // UChar to LChar when 8 bit strings are turned on.
     433    size_t currentSize = m_buffer16.size();
     434    m_buffer16.grow(currentSize + length);
     435    UChar* rawBuffer = m_buffer16.data() + currentSize;
     436
     437    for (size_t i = 0; i < length; i++) {
     438        T c = p[i];
     439        ASSERT(c >= 0);
     440        ASSERT(c <= 0xFF);
     441        rawBuffer[i] = c;
     442    }
     443}
     444
     445template <typename T>
     446inline void Lexer<T>::append16(const LChar* p, size_t length)
     447{
     448    size_t currentSize = m_buffer16.size();
     449    m_buffer16.grow(currentSize + length);
     450    UChar* rawBuffer = m_buffer16.data() + currentSize;
     451
     452    for (size_t i = 0; i < length; i++)
     453        rawBuffer[i] = p[i];
     454}
     455
     456template <typename T>
     457inline void Lexer<T>::record16(T c)
    422458{
    423459    m_buffer16.append(c);
    424460}
    425461
    426 inline void Lexer::record16(int c)
     462template <typename T>
     463inline void Lexer<T>::record16(int c)
    427464{
    428465    ASSERT(c >= 0);
    429466    ASSERT(c <= USHRT_MAX);
    430     record16(UChar(static_cast<unsigned short>(c)));
    431 }
    432 
    433 template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer::parseIdentifier(JSTokenData* tokenData, unsigned lexType, bool strictMode)
     467    m_buffer16.append(static_cast<UChar>(c));
     468}
     469
     470template <>
     471    template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<LChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
    434472{
    435473    const ptrdiff_t remaining = m_codeEnd - m_code;
    436     if ((remaining >= maxTokenLength) && !(lexType & IgnoreReservedWords)) {
     474    if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {
    437475        JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);
    438476        if (keyword != IDENT) {
    … …  
    441479        }
    442480    }
     481
     482    const LChar* identifierStart = currentCharacter();
     483   
     484    while (isIdentPart(m_current))
     485        shift();
     486   
     487    if (UNLIKELY(m_current == '\\')) {
     488        setOffsetFromCharOffset(identifierStart);
     489        return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);
     490    }
     491
     492    const Identifier* ident = 0;
     493   
     494    if (shouldCreateIdentifier) {
     495        int identifierLength = currentCharacter() - identifierStart;
     496        ident = makeIdentifier(identifierStart, identifierLength);
     497
     498        tokenData->ident = ident;
     499    } else
     500        tokenData->ident = 0;
     501
     502    m_delimited = false;
     503
     504    if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
     505        ASSERT(shouldCreateIdentifier);
     506        if (remaining < maxTokenLength) {
     507            const HashEntry* entry = m_globalData->keywords->getKeyword(*ident);
     508            ASSERT((remaining < maxTokenLength) || !entry);
     509            if (!entry)
     510                return IDENT;
     511            JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
     512            return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
     513        }
     514        return IDENT;
     515    }
     516
     517    return IDENT;
     518}
     519
     520template <>
     521template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<UChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
     522{
     523    const ptrdiff_t remaining = m_codeEnd - m_code;
     524    if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {
     525        JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);
     526        if (keyword != IDENT) {
     527            ASSERT((!shouldCreateIdentifier) || tokenData->ident);
     528            return keyword == RESERVED_IF_STRICT && !strictMode ? IDENT : keyword;
     529        }
     530    }
    443531    const UChar* identifierStart = currentCharacter();
     532
     533    UChar orAllChars = 0;
     534   
     535    while (isIdentPart(m_current)) {
     536        orAllChars |= m_current;
     537        shift();
     538    }
     539   
     540    if (UNLIKELY(m_current == '\\')) {
     541        setOffsetFromCharOffset(identifierStart);
     542        return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);
     543    }
     544
     545    bool isAll8Bit = false;
     546
     547#if 0 // FIXME: Remove this #if when 8 bit strings are turned on.
     548    if (!(orAllChars & ~0xff))
     549        isAll8Bit = true;
     550#endif
     551
     552    const Identifier* ident = 0;
     553   
     554    if (shouldCreateIdentifier) {
     555        int identifierLength = currentCharacter() - identifierStart;
     556        if (isAll8Bit)
     557            ident = makeIdentifierLCharFromUChar(identifierStart, identifierLength);
     558        else
     559            ident = makeIdentifier(identifierStart, identifierLength);
     560       
     561        tokenData->ident = ident;
     562    } else
     563        tokenData->ident = 0;
     564   
     565    m_delimited = false;
     566   
     567    if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
     568        ASSERT(shouldCreateIdentifier);
     569        if (remaining < maxTokenLength) {
     570            const HashEntry* entry = m_globalData->keywords->getKeyword(*ident);
     571            ASSERT((remaining < maxTokenLength) || !entry);
     572            if (!entry)
     573                return IDENT;
     574            JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
     575            return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
     576        }
     577        return IDENT;
     578    }
     579
     580    return IDENT;
     581}
     582
     583template <typename T>
     584template <bool shouldCreateIdentifier> JSTokenType Lexer<T>::parseIdentifierSlowCase(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
     585{
     586    const ptrdiff_t remaining = m_codeEnd - m_code;
     587    const T* identifierStart = currentCharacter();
    444588    bool bufferRequired = false;
    445589
    … …  
    465609        if (UNLIKELY(m_buffer16.size() ? !isIdentPart(character) : !isIdentStart(character)))
    466610            return ERRORTOK;
    467         if  (shouldCreateIdentifier)
     611        if (shouldCreateIdentifier)
    468612            record16(character);
    469613        identifierStart = currentCharacter();
    470614    }
    471    
     615
    472616    int identifierLength;
    473617    const Identifier* ident = 0;
    474618    if (shouldCreateIdentifier) {
    475         if (!bufferRequired)
     619        if (!bufferRequired) {
    476620            identifierLength = currentCharacter() - identifierStart;
    477         else {
     621            ident = makeIdentifier(identifierStart, identifierLength);
     622        } else {
    478623            if (identifierStart != currentCharacter())
    479624                m_buffer16.append(identifierStart, currentCharacter() - identifierStart);
    480             identifierStart = m_buffer16.data();
    481             identifierLength = m_buffer16.size();
    482         }
    483 
    484         ident = makeIdentifier(identifierStart, identifierLength);
     625            ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
     626        }
     627
    485628        tokenData->ident = ident;
    486629    } else
    … …  
    489632    m_delimited = false;
    490633
    491     if (LIKELY(!bufferRequired && !(lexType & IgnoreReservedWords))) {
     634    if (LIKELY(!bufferRequired && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
    492635        ASSERT(shouldCreateIdentifier);
    493636        // Keywords must not be recognized if there was an \uXXXX in the identifier.
    … …  
    507650}
    508651
    509 template <bool shouldBuildStrings> ALWAYS_INLINE bool Lexer::parseString(JSTokenData* tokenData, bool strictMode)
    510 {
     652template <typename T>
     653template <bool shouldBuildStrings> ALWAYS_INLINE bool Lexer<T>::parseString(JSTokenData* tokenData, bool strictMode)
     654{
     655    // FIXME: Change record16 and m_buffer16 to record8 and m_buffer8 below when
     656    // 8 bit strings are turned on.
     657    int startingOffset = currentOffset();
     658    int startingLineNumber = lineNumber();
    511659    int stringQuoteCharacter = m_current;
    512660    shift();
    513661
    514     const UChar* stringStart = currentCharacter();
     662    const T* stringStart = currentCharacter();
     663
     664    while (m_current != stringQuoteCharacter) {
     665        if (UNLIKELY((m_current == '\\'))) {
     666            if (stringStart != currentCharacter() && shouldBuildStrings)
     667                append8(stringStart, currentCharacter() - stringStart);
     668            shift();
     669
     670            int escape = singleEscape(m_current);
     671
     672            // Most common escape sequences first
     673            if (escape) {
     674                if (shouldBuildStrings)
     675                    record16(escape); // FIXME: Change to record8
     676                shift();
     677            } else if (UNLIKELY(isLineTerminator(m_current)))
     678                shiftLineTerminator();
     679            else if (m_current == 'x') {
     680                shift();
     681                if (isASCIIHexDigit(m_current) && isASCIIHexDigit(peek(1))) {
     682                    int prev = m_current;
     683                    shift();
     684                    if (shouldBuildStrings)
     685                        record16(convertHex(prev, m_current)); // FIXME: Change to record8
     686                    shift();
     687                } else if (shouldBuildStrings)
     688                    record16('x'); // FIXME: Change to record8
     689            } else {
     690                setOffset(startingOffset);
     691                setLineNumber(startingLineNumber);
     692                m_buffer16.resize(0); // FIXME: Change to m_buffer8
     693                return parseStringSlowCase<shouldBuildStrings>(tokenData, strictMode);
     694            }
     695            stringStart = currentCharacter();
     696            continue;
     697        }
     698
     699        if (UNLIKELY(((m_current > 0xff) || (m_current < 0xe)))) {
     700            setOffset(startingOffset);
     701            setLineNumber(startingLineNumber);
     702            m_buffer16.resize(0); // FIXME: Change to m_buffer8
     703            return parseStringSlowCase<shouldBuildStrings>(tokenData, strictMode);
     704        }
     705
     706        shift();
     707    }
     708
     709    if (currentCharacter() != stringStart && shouldBuildStrings)
     710        append8(stringStart, currentCharacter() - stringStart);
     711    if (shouldBuildStrings) {
     712        // FIXME: Change to m_buffer8
     713        tokenData->ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
     714        // FIXME: Change to m_buffer8
     715        m_buffer16.resize(0);
     716    } else
     717        tokenData->ident = 0;
     718
     719    return true;
     720}
     721
     722template <typename T>
     723template <bool shouldBuildStrings> bool Lexer<T>::parseStringSlowCase(JSTokenData* tokenData, bool strictMode)
     724{
     725    int stringQuoteCharacter = m_current;
     726    shift();
     727
     728    const T* stringStart = currentCharacter();
    515729
    516730    while (m_current != stringQuoteCharacter) {
    517731        if (UNLIKELY(m_current == '\\')) {
    518732            if (stringStart != currentCharacter() && shouldBuildStrings)
    519                 m_buffer16.append(stringStart, currentCharacter() - stringStart);
     733                append16(stringStart, currentCharacter() - stringStart);
    520734            shift();
    521735
    … …  
    524738            // Most common escape sequences first
    525739            if (escape) {
    526                  if (shouldBuildStrings)
    527                      record16(escape);
     740                if (shouldBuildStrings)
     741                    record16(escape);
    528742                shift();
    529743            } else if (UNLIKELY(isLineTerminator(m_current)))
    … …  
    609823
    610824    if (currentCharacter() != stringStart && shouldBuildStrings)
    611         m_buffer16.append(stringStart, currentCharacter() - stringStart);
     825        append16(stringStart, currentCharacter() - stringStart);
    612826    if (shouldBuildStrings)
    613827        tokenData->ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
    … …  
    619833}
    620834
    621 ALWAYS_INLINE void Lexer::parseHex(double& returnValue)
     835template <typename T>
     836ALWAYS_INLINE void Lexer<T>::parseHex(double& returnValue)
    622837{
    623838    // Optimization: most hexadecimal values fit into 4 bytes.
    … …  
    658873}
    659874
    660 ALWAYS_INLINE bool Lexer::parseOctal(double& returnValue)
     875template <typename T>
     876ALWAYS_INLINE bool Lexer<T>::parseOctal(double& returnValue)
    661877{
    662878    // Optimization: most octal values fit into 4 bytes.
    … …  
    665881    // Temporary buffer for the digits. Makes easier
    666882    // to reconstruct the input characters when needed.
    667     char digits[10];
     883    LChar digits[10];
    668884
    669885    do {
    … …  
    694910}
    695911
    696 ALWAYS_INLINE bool Lexer::parseDecimal(double& returnValue)
     912template <typename T>
     913ALWAYS_INLINE bool Lexer<T>::parseDecimal(double& returnValue)
    697914{
    698915    // Optimization: most decimal values fit into 4 bytes.
    … …  
    705922        // Temporary buffer for the digits. Makes easier
    706923        // to reconstruct the input characters when needed.
    707         char digits[10];
     924        LChar digits[10];
    708925
    709926        do {
    … …  
    731948}
    732949
    733 ALWAYS_INLINE void Lexer::parseNumberAfterDecimalPoint()
     950template <typename T>
     951ALWAYS_INLINE void Lexer<T>::parseNumberAfterDecimalPoint()
    734952{
    735953    record8('.');
    … …  
    740958}
    741959
    742 ALWAYS_INLINE bool Lexer::parseNumberAfterExponentIndicator()
     960template <typename T>
     961ALWAYS_INLINE bool Lexer<T>::parseNumberAfterExponentIndicator()
    743962{
    744963    record8('e');
    … …  
    759978}
    760979
    761 ALWAYS_INLINE bool Lexer::parseMultilineComment()
     980template <typename T>
     981ALWAYS_INLINE bool Lexer<T>::parseMultilineComment()
    762982{
    763983    while (true) {
    … …  
    7811001}
    7821002
    783 bool Lexer::nextTokenIsColon()
    784 {
    785     const UChar* code = m_code;
     1003template <typename T>
     1004bool Lexer<T>::nextTokenIsColon()
     1005{
     1006    const T* code = m_code;
    7861007    while (code < m_codeEnd && (isWhiteSpace(*code) || isLineTerminator(*code)))
    7871008        code++;
    788        
     1009   
    7891010    return code < m_codeEnd && *code == ':';
    7901011}
    7911012
    792 JSTokenType Lexer::lex(JSTokenData* tokenData, JSTokenInfo* tokenInfo, unsigned lexType, bool strictMode)
     1013template <typename T>
     1014JSTokenType Lexer<T>::lex(JSTokenData* tokenData, JSTokenInfo* tokenInfo, unsigned lexerFlags, bool strictMode)
    7931015{
    7941016    ASSERT(!m_error);
    … …  
    10961318                // Null-terminate string for strtod.
    10971319                m_buffer8.append('\0');
    1098                 tokenData->doubleValue = WTF::strtod(m_buffer8.data(), 0);
     1320                tokenData->doubleValue = WTF::strtod(reinterpret_cast<const char*>(m_buffer8.data()), 0);
    10991321            }
    11001322            token = NUMBER;
    … …  
    11101332        break;
    11111333    case CharacterQuote:
    1112         if (lexType & DontBuildStrings) {
     1334        if (lexerFlags & LexerFlagsDontBuildStrings) {
    11131335            if (UNLIKELY(!parseString<false>(tokenData, strictMode)))
    11141336                goto returnError;
    … …  
    11251347        // Fall through into CharacterBackSlash.
    11261348    case CharacterBackSlash:
    1127         if (lexType & DontBuildKeywords)
    1128             token = parseIdentifier<false>(tokenData, lexType, strictMode);
     1349        if (lexerFlags & LexexFlagsDontBuildKeywords)
     1350            token = parseIdentifier<false>(tokenData, lexerFlags, strictMode);
    11291351        else
    1130             token = parseIdentifier<true>(tokenData, lexType, strictMode);
     1352            token = parseIdentifier<true>(tokenData, lexerFlags, strictMode);
    11311353        break;
    11321354    case CharacterLineTerminator:
    … …  
    11791401}
    11801402
    1181 bool Lexer::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix)
     1403template <typename T>
     1404bool Lexer<T>::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix)
    11821405{
    11831406    ASSERT(m_buffer16.isEmpty());
    … …  
    12401463}
    12411464
    1242 bool Lexer::skipRegExp()
     1465template <typename T>
     1466bool Lexer<T>::skipRegExp()
    12431467{
    12441468    bool lastWasEscape = false;
    … …  
    12801504}
    12811505
    1282 void Lexer::clear()
     1506template <typename T>
     1507void Lexer<T>::clear()
    12831508{
    12841509    m_arena = 0;
    12851510
    1286     Vector<char> newBuffer8;
     1511    Vector<LChar> newBuffer8;
    12871512    m_buffer8.swap(newBuffer8);
    12881513
    … …  
    12931518}
    12941519
    1295 SourceCode Lexer::sourceCode(int openBrace, int closeBrace, int firstLine)
     1520template <typename T>
     1521SourceCode Lexer<T>::sourceCode(int openBrace, int closeBrace, int firstLine)
    12961522{
    12971523    ASSERT(m_source->provider()->data()[openBrace] == '{');
    … …  
    13001526}
    13011527
     1528// Instantiate the two flavors of Lexer we need instead of putting most of this file in Lexer.h
     1529template class Lexer<LChar>;
     1530template class Lexer<UChar>;
     1531
    13021532} // namespace JSC
Note: See TracChangeset for help on using the changeset viewer.