Package org.mozilla.javascript
Class TokenStream
- java.lang.Object
-
- org.mozilla.javascript.TokenStream
-
class TokenStream extends java.lang.Object
This class implements the JavaScript scanner. It is based on the C source files jsscan.c and jsscan.h in the jsref package.- See Also:
Parser
-
-
Field Summary
Fields Modifier and Type Field Description private ObjToIntMap
allStrings
private static char
BYTE_ORDER_MARK
private int
commentCursor
private java.lang.String
commentPrefix
(package private) Token.CommentType
commentType
(package private) int
cursor
private boolean
dirtyLine
private static int
EOF_CHAR
private boolean
hitEOF
private boolean
isBinary
private boolean
isHex
private boolean
isOctal
private boolean
isOldOctal
private int
lineEndChar
(package private) int
lineno
private int
lineStart
private double
number
private Parser
parser
private int
quoteChar
(package private) java.lang.String
regExpFlags
private char[]
sourceBuffer
(package private) int
sourceCursor
private int
sourceEnd
private java.io.Reader
sourceReader
private java.lang.String
sourceString
private java.lang.String
string
private char[]
stringBuffer
private int
stringBufferTop
(package private) int
tokenBeg
(package private) int
tokenEnd
private int[]
ungetBuffer
private int
ungetCursor
private boolean
xmlIsAttribute
private boolean
xmlIsTagContent
private int
xmlOpenTagsCount
-
Constructor Summary
Constructors Constructor Description TokenStream(Parser parser, java.io.Reader sourceReader, java.lang.String sourceString, int lineno)
-
Method Summary
All Methods Static Methods Instance Methods Concrete Methods Modifier and Type Method Description private void
addToString(int c)
private boolean
canUngetChar()
private int
charAt(int index)
private static java.lang.String
convertLastCharToHex(java.lang.String str)
(package private) boolean
eof()
private boolean
fillSourceBuffer()
(package private) java.lang.String
getAndResetCurrentComment()
private int
getChar()
private int
getChar(boolean skipFormattingChars)
private int
getCharIgnoreLineEnd()
Token.CommentType
getCommentType()
Return the type of the last scanned comment.int
getCursor()
Return the current position of the scanner cursor.(package private) int
getFirstXMLToken()
(package private) java.lang.String
getLine()
(package private) java.lang.String
getLine(int position, int[] linep)
(package private) int
getLineno()
(package private) int
getNextXMLToken()
(package private) double
getNumber()
(package private) int
getOffset()
Returns the offset into the current line.(package private) char
getQuoteChar()
(package private) java.lang.String
getSourceString()
(package private) java.lang.String
getString()
private java.lang.String
getStringFromBuffer()
(package private) int
getToken()
int
getTokenBeg()
Return the absolute source offset of the last scanned token.int
getTokenEnd()
Return the absolute source end-offset of the last scanned token.int
getTokenLength()
Return tokenEnd - tokenBegprivate static boolean
isAlpha(int c)
(package private) static boolean
isDigit(int c)
private static boolean
isJSFormatChar(int c)
(package private) static boolean
isJSSpace(int c)
(package private) static boolean
isKeyword(java.lang.String s, int version, boolean isStrict)
private boolean
isMarkingComment()
(package private) boolean
isNumberBinary()
(package private) boolean
isNumberHex()
(package private) boolean
isNumberOctal()
(package private) boolean
isNumberOldOctal()
(package private) boolean
isXMLAttribute()
private void
markCommentStart()
private void
markCommentStart(java.lang.String prefix)
private boolean
matchChar(int test)
private int
peekChar()
(package private) java.lang.String
readAndClearRegExpFlags()
private boolean
readCDATA()
private boolean
readEntity()
private boolean
readPI()
private boolean
readQuotedString(int quote)
(package private) void
readRegExp(int startToken)
Parser calls the method when it gets / or /= in literal context.private boolean
readXmlComment()
private void
skipLine()
private static int
stringToKeyword(java.lang.String name, int version, boolean isStrict)
private static int
stringToKeywordForES(java.lang.String name, boolean isStrict)
ECMAScript 6.private static int
stringToKeywordForJS(java.lang.String name)
JavaScript 1.8 and earlierprivate java.lang.String
substring(int beginIndex, int endIndex)
(package private) java.lang.String
tokenToString(int token)
private void
ungetChar(int c)
private void
ungetCharIgnoreLineEnd(int c)
-
-
-
Field Detail
-
EOF_CHAR
private static final int EOF_CHAR
- See Also:
- Constant Field Values
-
BYTE_ORDER_MARK
private static final char BYTE_ORDER_MARK
- See Also:
- Constant Field Values
-
dirtyLine
private boolean dirtyLine
-
regExpFlags
java.lang.String regExpFlags
-
string
private java.lang.String string
-
number
private double number
-
isBinary
private boolean isBinary
-
isOldOctal
private boolean isOldOctal
-
isOctal
private boolean isOctal
-
isHex
private boolean isHex
-
quoteChar
private int quoteChar
-
stringBuffer
private char[] stringBuffer
-
stringBufferTop
private int stringBufferTop
-
allStrings
private ObjToIntMap allStrings
-
ungetBuffer
private final int[] ungetBuffer
-
ungetCursor
private int ungetCursor
-
hitEOF
private boolean hitEOF
-
lineStart
private int lineStart
-
lineEndChar
private int lineEndChar
-
lineno
int lineno
-
sourceString
private java.lang.String sourceString
-
sourceReader
private java.io.Reader sourceReader
-
sourceBuffer
private char[] sourceBuffer
-
sourceEnd
private int sourceEnd
-
sourceCursor
int sourceCursor
-
cursor
int cursor
-
tokenBeg
int tokenBeg
-
tokenEnd
int tokenEnd
-
commentType
Token.CommentType commentType
-
xmlIsAttribute
private boolean xmlIsAttribute
-
xmlIsTagContent
private boolean xmlIsTagContent
-
xmlOpenTagsCount
private int xmlOpenTagsCount
-
parser
private Parser parser
-
commentPrefix
private java.lang.String commentPrefix
-
commentCursor
private int commentCursor
-
-
Constructor Detail
-
TokenStream
TokenStream(Parser parser, java.io.Reader sourceReader, java.lang.String sourceString, int lineno)
-
-
Method Detail
-
tokenToString
java.lang.String tokenToString(int token)
-
isKeyword
static boolean isKeyword(java.lang.String s, int version, boolean isStrict)
-
stringToKeyword
private static int stringToKeyword(java.lang.String name, int version, boolean isStrict)
-
stringToKeywordForJS
private static int stringToKeywordForJS(java.lang.String name)
JavaScript 1.8 and earlier
-
stringToKeywordForES
private static int stringToKeywordForES(java.lang.String name, boolean isStrict)
ECMAScript 6.
-
getSourceString
final java.lang.String getSourceString()
-
getLineno
final int getLineno()
-
getString
final java.lang.String getString()
-
getQuoteChar
final char getQuoteChar()
-
getNumber
final double getNumber()
-
isNumberBinary
final boolean isNumberBinary()
-
isNumberOldOctal
final boolean isNumberOldOctal()
-
isNumberOctal
final boolean isNumberOctal()
-
isNumberHex
final boolean isNumberHex()
-
eof
final boolean eof()
-
getToken
final int getToken() throws java.io.IOException
- Throws:
java.io.IOException
-
isAlpha
private static boolean isAlpha(int c)
-
isDigit
static boolean isDigit(int c)
-
isJSSpace
static boolean isJSSpace(int c)
-
isJSFormatChar
private static boolean isJSFormatChar(int c)
-
readRegExp
void readRegExp(int startToken) throws java.io.IOException
Parser calls the method when it gets / or /= in literal context.- Throws:
java.io.IOException
-
readAndClearRegExpFlags
java.lang.String readAndClearRegExpFlags()
-
isXMLAttribute
boolean isXMLAttribute()
-
getFirstXMLToken
int getFirstXMLToken() throws java.io.IOException
- Throws:
java.io.IOException
-
getNextXMLToken
int getNextXMLToken() throws java.io.IOException
- Throws:
java.io.IOException
-
readQuotedString
private boolean readQuotedString(int quote) throws java.io.IOException
- Throws:
java.io.IOException
-
readXmlComment
private boolean readXmlComment() throws java.io.IOException
- Throws:
java.io.IOException
-
readCDATA
private boolean readCDATA() throws java.io.IOException
- Throws:
java.io.IOException
-
readEntity
private boolean readEntity() throws java.io.IOException
- Throws:
java.io.IOException
-
readPI
private boolean readPI() throws java.io.IOException
- Throws:
java.io.IOException
-
getStringFromBuffer
private java.lang.String getStringFromBuffer()
-
addToString
private void addToString(int c)
-
canUngetChar
private boolean canUngetChar()
-
ungetChar
private void ungetChar(int c)
-
matchChar
private boolean matchChar(int test) throws java.io.IOException
- Throws:
java.io.IOException
-
peekChar
private int peekChar() throws java.io.IOException
- Throws:
java.io.IOException
-
getChar
private int getChar() throws java.io.IOException
- Throws:
java.io.IOException
-
getChar
private int getChar(boolean skipFormattingChars) throws java.io.IOException
- Throws:
java.io.IOException
-
getCharIgnoreLineEnd
private int getCharIgnoreLineEnd() throws java.io.IOException
- Throws:
java.io.IOException
-
ungetCharIgnoreLineEnd
private void ungetCharIgnoreLineEnd(int c)
-
skipLine
private void skipLine() throws java.io.IOException
- Throws:
java.io.IOException
-
getOffset
final int getOffset()
Returns the offset into the current line.
-
charAt
private final int charAt(int index)
-
substring
private final java.lang.String substring(int beginIndex, int endIndex)
-
getLine
final java.lang.String getLine()
-
getLine
final java.lang.String getLine(int position, int[] linep)
-
fillSourceBuffer
private boolean fillSourceBuffer() throws java.io.IOException
- Throws:
java.io.IOException
-
getCursor
public int getCursor()
Return the current position of the scanner cursor.
-
getTokenBeg
public int getTokenBeg()
Return the absolute source offset of the last scanned token.
-
getTokenEnd
public int getTokenEnd()
Return the absolute source end-offset of the last scanned token.
-
getTokenLength
public int getTokenLength()
Return tokenEnd - tokenBeg
-
getCommentType
public Token.CommentType getCommentType()
Return the type of the last scanned comment.- Returns:
- type of last scanned comment, or 0 if none have been scanned.
-
markCommentStart
private void markCommentStart()
-
markCommentStart
private void markCommentStart(java.lang.String prefix)
-
isMarkingComment
private boolean isMarkingComment()
-
getAndResetCurrentComment
final java.lang.String getAndResetCurrentComment()
-
convertLastCharToHex
private static java.lang.String convertLastCharToHex(java.lang.String str)
-
-