Changeset 244038 in webkit for trunk/Source/JavaScriptCore/parser/Lexer.cpp

Timestamp:

Apr 8, 2019, 1:43:18 PM (6 years ago)

Author:

[email protected]

Message:

Unreviewed, rolling in r243948 with test fix
​https://bugs.webkit.org/show_bug.cgi?id=196486

JSTests:

• stress/arrow-function-and-use-strict-directive.js: Added.
• stress/arrow-function-syntax.js: Added.

(checkSyntax):
(checkSyntaxError):

Source/JavaScriptCore:

• parser/ASTBuilder.h:

(JSC::ASTBuilder::createString):

• parser/Lexer.cpp:

(JSC::Lexer<T>::parseMultilineComment):
(JSC::Lexer<T>::lexWithoutClearingLineTerminator):
(JSC::Lexer<T>::lex): Deleted.

• parser/Lexer.h:

(JSC::Lexer::hasLineTerminatorBeforeToken const):
(JSC::Lexer::setHasLineTerminatorBeforeToken):
(JSC::Lexer<T>::lex):
(JSC::Lexer::prevTerminator const): Deleted.
(JSC::Lexer::setTerminator): Deleted.

• parser/Parser.cpp:

(JSC::Parser<LexerType>::allowAutomaticSemicolon):
(JSC::Parser<LexerType>::parseSingleFunction):
(JSC::Parser<LexerType>::parseStatementListItem):
(JSC::Parser<LexerType>::maybeParseAsyncFunctionDeclarationStatement):
(JSC::Parser<LexerType>::parseFunctionInfo):
(JSC::Parser<LexerType>::parseClass):
(JSC::Parser<LexerType>::parseExportDeclaration):
(JSC::Parser<LexerType>::parseAssignmentExpression):
(JSC::Parser<LexerType>::parseYieldExpression):
(JSC::Parser<LexerType>::parseProperty):
(JSC::Parser<LexerType>::parsePrimaryExpression):
(JSC::Parser<LexerType>::parseMemberExpression):

• parser/Parser.h:

(JSC::Parser::nextWithoutClearingLineTerminator):
(JSC::Parser::lexCurrentTokenAgainUnderCurrentContext):
(JSC::Parser::internalSaveLexerState):
(JSC::Parser::restoreLexerState):

LayoutTests:

The test relied on the wrong EOF token's offset. This patch also fixes the test.

• inspector/runtime/parse-expected.txt:
• inspector/runtime/parse.html:

File:

• trunk/Source/JavaScriptCore/parser/Lexer.cpp (modified) (8 diffs)

trunk/Source/JavaScriptCore/parser/Lexer.cpp

@@ -1692 +1692 @@
         if (isLineTerminator(m_current)) {
             shiftLineTerminator();
-            m_terminator = true;
+            m_hasLineTerminatorBeforeToken = true;
         } else
             shift();
@@ -1771 +1771 @@
 
 template <typename T>
-JSTokenType Lexer<T>::lex(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
+JSTokenType Lexer<T>::lexWithoutClearingLineTerminator(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
 {
     JSTokenData* tokenData = &tokenRecord->m_data;
@@ -1782 +1782 @@
 
     JSTokenType token = ERRORTOK;
-    m_terminator = false;
 
 start:
     skipWhitespace();
 
-    if (atEnd())
-        return EOFTOK;
-    
     tokenLocation->startOffset = currentOffset();
     ASSERT(currentOffset() >= currentLineStartOffset());
     tokenRecord->m_startPosition = currentPosition();
+
+    if (atEnd()) {
+        token = EOFTOK;
+        goto returnToken;
+    }
 
     CharacterType type;
@@ -1903 +1904 @@
         if (m_current == '+') {
             shift();
-            token = (!m_terminator) ? PLUSPLUS : AUTOPLUSPLUS;
+            token = (!m_hasLineTerminatorBeforeToken) ? PLUSPLUS : AUTOPLUSPLUS;
             break;
         }
@@ -1917 +1918 @@
         if (m_current == '-') {
             shift();
-            if ((m_atLineStart || m_terminator) && m_current == '>') {
+            if ((m_atLineStart || m_hasLineTerminatorBeforeToken) && m_current == '>') {
                 if (m_scriptMode == JSParserScriptMode::Classic) {
                     shift();
@@ -1923 +1924 @@
                 }
             }
-            token = (!m_terminator) ? MINUSMINUS : AUTOMINUSMINUS;
+            token = (!m_hasLineTerminatorBeforeToken) ? MINUSMINUS : AUTOMINUSMINUS;
             break;
         }
@@ -2294 +2295 @@
         shiftLineTerminator();
         m_atLineStart = true;
-        m_terminator = true;
+        m_hasLineTerminatorBeforeToken = true;
         m_lineStart = m_code;
         goto start;
@@ -2334 +2335 @@
 
         while (!isLineTerminator(m_current)) {
-            if (atEnd())
-                return EOFTOK;
+            if (atEnd()) {
+                token = EOFTOK;
+                fillTokenInfo(tokenRecord, token, lineNumber, endOffset, lineStartOffset, endPosition);
+                return token;
+            }
             shift();
         }
         shiftLineTerminator();
         m_atLineStart = true;
-        m_terminator = true;
+        m_hasLineTerminatorBeforeToken = true;
         m_lineStart = m_code;
         if (!lastTokenWasRestrKeyword())