Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js

/*
 * Copyright (C) 2009 Google Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Generate js file as follows:
re2c -isc Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js \
  | sed 's|^yy\([^:]*\)*\:|case \1:|' \
  | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \
  | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \
  | sed 's|[*]cursor|this._charAt(cursor)|' \
  | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \
  | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \
  | sed 's|yych <= \(0x[0-9a-fA-F]*\)|yych \<\= String.fromCharCode(\1)|' \
  | sed 's|unsigned\ int|var|' \
  | sed 's|var\ yych|case 1: var yych|' > Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.js
 */

/**
 * @constructor
 * @extends {WebInspector.SourceTokenizer}
 */
WebInspector.SourceJavaScriptTokenizer = function()
{
    WebInspector.SourceTokenizer.call(this);

    this._lexConditions = {
        DIV: 0,
        NODIV: 1,
        COMMENT: 2,
        DSTRING: 3,
        SSTRING: 4,
        REGEX: 5
    };

    this.case_DIV = 1000;
    this.case_NODIV = 1001;
    this.case_COMMENT = 1002;
    this.case_DSTRING = 1003;
    this.case_SSTRING = 1004;
    this.case_REGEX = 1005;

    this.condition = this.createInitialCondition();
}

WebInspector.SourceJavaScriptTokenizer.Keywords = [
        "null", "true", "false", "break", "case", "catch", "const", "default", "finally", "for",
        "instanceof", "new", "var", "continue", "function", "return", "void", "delete", "if",
        "this", "do", "while", "else", "in", "switch", "throw", "try", "typeof", "debugger",
        "class", "enum", "export", "extends", "import", "super", "get", "set", "with"
    ].keySet();

WebInspector.SourceJavaScriptTokenizer.prototype = {
    createInitialCondition: function()
    {
        return { lexCondition: this._lexConditions.NODIV };
    },

    nextToken: function(cursor)
    {
        var cursorOnEnter = cursor;
        var gotoCase = 1;
        var YYMARKER;
        while (1) {
            switch (gotoCase)
            // Following comment is replaced with generated state machine.
            /*!re2c
                re2c:define:YYCTYPE  = "var";
                re2c:define:YYCURSOR = cursor;
                re2c:define:YYGETCONDITION = "this.getLexCondition";
                re2c:define:YYSETCONDITION = "this.setLexCondition";
                re2c:condprefix = "case this.case_";
                re2c:condenumprefix = "this._lexConditions.";
                re2c:yyfill:enable = 0;
                re2c:labelprefix = "case ";
                re2c:indent:top = 2;
                re2c:indent:string = "    ";

                LineComment = "//" [^\r\n]*;
                CommentContent = ([^*\r\n] | ("*"+[^/*]))*;
                Comment = "/*" CommentContent "*"+ "/";
                CommentStart = "/*" CommentContent [\r\n];
                CommentEnd = CommentContent "*"+ "/";

                DecimalDigit = [0-9];
                NonZeroDigit = [1-9];
                OctalDigit = [0-7];
                HexDigit = [0-9a-fA-F];
                SignedInteger = ("+"|"-")? DecimalDigit+;
                ExponentPart = ("e" | "E") SignedInteger;
                DecimalIntegerLiteral = "0" | NonZeroDigit DecimalDigit*;
                DecimalLiteral = DecimalIntegerLiteral "." DecimalDigit* ExponentPart? | "." DecimalDigit+ ExponentPart? | DecimalIntegerLiteral ExponentPart?;
                HexIntegerLiteral = "0" ("x"|"X") HexDigit+;
                OctalIntegerLiteral = "0" OctalDigit+;
                NumericLiteral = DecimalLiteral | HexIntegerLiteral | OctalIntegerLiteral;

                Punctuation = [\!\%\&\(\*\+\,\-\.\:\;\<\=\>\?\[\]\^\{\|\}\~] | "!=" | "!==" | "%=" | "&&" | "&=" | "*=" | "++" | "+=" | "--" | "-=" | "<<" | "<<="  | "<=" | "==" | "===" | ">=" | ">>" | ">>=" | ">>>" | ">>>=" | "^=" | "|=" | "||";
                Division = "/" | "/=";
                RightParen = ")";

                Letter = [a-zA-Z\x80-\xFF];
                UnicodeEscapeSequence = "\\u" HexDigit HexDigit HexDigit HexDigit;

                IdentifierStart = Letter | "_" | "$" | UnicodeEscapeSequence;
                IdentifierPart = IdentifierStart | DecimalDigit;
                Identifier = IdentifierStart IdentifierPart *;

                DoubleStringContent = ([^\r\n\"\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
                SingleStringContent = ([^\r\n\'\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
                StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleStringContent "'";
                DoubleStringStart = "\"" DoubleStringContent "\\" [\r\n];
                DoubleStringEnd = DoubleStringContent "\"";
                SingleStringStart = "'" SingleStringContent "\\" [\r\n];
                SingleStringEnd = SingleStringContent "'";

                BackslashSequence = "\\" [^\r\n];
                RegexSet = "[" ([^\r\n*\\/] | BackslashSequence)* "]";
                RegexFirstChar = [^\r\n*\\/\[\]] | BackslashSequence | RegexSet;
                RegexChar = [^\r\n\\/\[\]] | BackslashSequence | RegexSet;
                RegexContent = RegexChar*;
                Regex = "/" RegexFirstChar RegexContent "/" [igm]*;
                RegexStart = "/" RegexFirstChar RegexContent "\\";
                RegexEnd = RegexContent "/" [igm]*;

                <DIV,NODIV> LineComment { this.tokenType = "javascript-comment"; return cursor; }
                <DIV,NODIV> Comment { this.tokenType = "javascript-comment"; return cursor; }
                <DIV,NODIV> CommentStart => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
                <COMMENT> CommentContent => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
                <COMMENT> CommentEnd => NODIV { this.tokenType = "javascript-comment"; return cursor; }

                <DIV,NODIV> StringLiteral { this.tokenType = "javascript-string"; return cursor; }
                <DIV,NODIV> DoubleStringStart => DSTRING { this.tokenType = "javascript-string"; return cursor; }
                <DSTRING> DoubleStringContent => DSTRING { this.tokenType = "javascript-string"; return cursor; }
                <DSTRING> DoubleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }
                <DIV,NODIV> SingleStringStart => SSTRING { this.tokenType = "javascript-string"; return cursor; }
                <SSTRING> SingleStringContent => SSTRING { this.tokenType = "javascript-string"; return cursor; }
                <SSTRING> SingleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }

                <NODIV> Regex { this.tokenType = "javascript-regexp"; return cursor; }
                <NODIV> RegexStart => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
                <REGEX> RegexContent => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
                <REGEX> RegexEnd => NODIV { this.tokenType = "javascript-regexp"; return cursor; }

                <DIV,NODIV> NumericLiteral => DIV { this.tokenType = "javascript-number"; return cursor; }
                <DIV,NODIV> Identifier => DIV
                {
                    var token = this._line.substring(cursorOnEnter, cursor);
                    if (WebInspector.SourceJavaScriptTokenizer.Keywords[token] === true && token !== "__proto__")
                        this.tokenType = "javascript-keyword";
                    else
                        this.tokenType = "javascript-ident";
                    return cursor;
                }
                <DIV,NODIV> RightParen => DIV { this.tokenType = null; return cursor; }
                <DIV,NODIV> Punctuation => NODIV
                {
                    var token = this._line.charAt(cursorOnEnter);
                    if (token === "{")
                        this.tokenType = "block-start";
                    else if (token === "}")
                        this.tokenType = "block-end";
                    else this.tokenType = null;
                    return cursor;
                }
                <DIV> Division => NODIV { this.tokenType = null; return cursor; }
                <*> [^] { this.tokenType = null; return cursor; }
            */
        }
    }
}

WebInspector.SourceJavaScriptTokenizer.prototype.__proto__ = WebInspector.SourceTokenizer.prototype;