Copyright (c) 2000, 2019 IBM Corporation and others. This program and the accompanying materials are made available under the terms of the Eclipse Public License 2.0 which accompanies this distribution, and is available at https://www.eclipse.org/legal/epl-2.0/ SPDX-License-Identifier: EPL-2.0 Contributors: IBM Corporation - initial API and implementation Stephan Herrmann - Contribution for bug 186342 - [compiler][null] Using annotations for null checking Jesper S Moller -. Contribution for bug 400830: [1.8][formatter] Code formatter for Java 8
/******************************************************************************* * Copyright (c) 2000, 2019 IBM Corporation and others. * * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at * https://www.eclipse.org/legal/epl-2.0/ * * SPDX-License-Identifier: EPL-2.0 * * Contributors: * IBM Corporation - initial API and implementation * Stephan Herrmann - Contribution for bug 186342 - [compiler][null] Using annotations for null checking * Jesper S Moller -. Contribution for bug 400830: [1.8][formatter] Code formatter for Java 8 *******************************************************************************/
package org.eclipse.jdt.internal.compiler.parser; import java.util.ArrayList; import java.util.List; import org.eclipse.jdt.core.compiler.CharOperation; import org.eclipse.jdt.core.compiler.InvalidInputException; import org.eclipse.jdt.internal.compiler.CompilationResult; import org.eclipse.jdt.internal.compiler.DefaultErrorHandlingPolicies; import org.eclipse.jdt.internal.compiler.ast.Statement; import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants; import org.eclipse.jdt.internal.compiler.impl.CompilerOptions; import org.eclipse.jdt.internal.compiler.problem.DefaultProblemFactory; import org.eclipse.jdt.internal.compiler.problem.ProblemReporter; import org.eclipse.jdt.internal.compiler.util.Util;
IMPORTANT NOTE: Internal Scanner implementation. It is mirrored in org.eclipse.jdt.core.compiler public package where it is API. The mirror implementation is using the backward compatible ITerminalSymbols constant definitions (stable with 2.0), whereas the internal implementation uses TerminalTokens which constant values reflect the latest parser generation state.
/** * IMPORTANT NOTE: Internal Scanner implementation. It is mirrored in * org.eclipse.jdt.core.compiler public package where it is API. * The mirror implementation is using the backward compatible ITerminalSymbols constant * definitions (stable with 2.0), whereas the internal implementation uses TerminalTokens * which constant values reflect the latest parser generation state. */
public class Scanner implements TerminalTokens { //public int newIdentCount = 0; /* APIs ares - getNextToken() which return the current type of the token (this value is not memorized by the scanner) - getCurrentTokenSource() which provides with the token "REAL" source (aka all unicode have been transformed into a correct char) - sourceStart gives the position into the stream - currentPosition-1 gives the sourceEnd position into the stream */ public long sourceLevel; public long complianceLevel; // 1.4 feature public boolean useAssertAsAnIndentifier = false; //flag indicating if processed source contains occurrences of keyword assert public boolean containsAssertKeyword = false; public boolean previewEnabled; // 1.5 feature public boolean useEnumAsAnIndentifier = false; public boolean recordLineSeparator = false; public char currentCharacter; public int startPosition; public int currentPosition; public int initialPosition, eofPosition; // after this position eof are generated instead of real token from the source public boolean skipComments = false; public boolean tokenizeComments = false; public boolean tokenizeWhiteSpace = false; //source should be viewed as a window (aka a part) //of a entire very large stream public char source[]; //unicode support public char[] withoutUnicodeBuffer; public int withoutUnicodePtr; //when == 0 ==> no unicode in the current token public boolean unicodeAsBackSlash = false; public boolean scanningFloatLiteral = false; //support for /** comments public final static int COMMENT_ARRAYS_SIZE = 30; public int[] commentStops = new int[COMMENT_ARRAYS_SIZE]; public int[] commentStarts = new int[COMMENT_ARRAYS_SIZE]; public int[] commentTagStarts = new int[COMMENT_ARRAYS_SIZE]; public int commentPtr = -1; // no comment test with commentPtr value -1 public int lastCommentLinePosition = -1; // task tag support public char[][] foundTaskTags = null; public char[][] foundTaskMessages; public char[][] foundTaskPriorities = null; public int[][] foundTaskPositions; public int foundTaskCount = 0; public char[][] taskTags = null; public char[][] taskPriorities = null; public boolean isTaskCaseSensitive = true; //diet parsing support - jump over some method body when requested public boolean diet = false; //support for the poor-line-debuggers .... //remember the position of the cr/lf public int[] lineEnds = new int[250]; public int linePtr = -1; public boolean wasAcr = false; public boolean fakeInModule = false; boolean inCase = false; /* package */ int yieldColons = -1; boolean breakPreviewAllowed = false;
The current context of the scanner w.r.t restricted keywords
/** * The current context of the scanner w.r.t restricted keywords * */
enum ScanContext { EXPECTING_KEYWORD, EXPECTING_IDENTIFIER, AFTER_REQUIRES, INACTIVE } protected ScanContext scanContext = null; protected boolean insideModuleInfo = false; public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$ public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$ public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$ public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$ public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$ public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$ public static final String INVALID_TEXTBLOCK = "Invalid_Textblock"; //$NON-NLS-1$ public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$ public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$ public static final String INVALID_LOW_SURROGATE = "Invalid_Low_Surrogate"; //$NON-NLS-1$ public static final String INVALID_HIGH_SURROGATE = "Invalid_High_Surrogate"; //$NON-NLS-1$ public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$ public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$ public static final String UNTERMINATED_TEXT_BLOCK = "Unterminated_Text_Block"; //$NON-NLS-1$ public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$ public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$ public static final String INVALID_DIGIT = "Invalid_Digit"; //$NON-NLS-1$ private static final int[] EMPTY_LINE_ENDS = Util.EMPTY_INT_ARRAY; public static final String INVALID_BINARY = "Invalid_Binary_Literal"; //$NON-NLS-1$ public static final String BINARY_LITERAL_NOT_BELOW_17 = "Binary_Literal_Not_Below_17"; //$NON-NLS-1$ public static final String ILLEGAL_HEXA_LITERAL = "Illegal_Hexa_Literal"; //$NON-NLS-1$ public static final String INVALID_UNDERSCORE = "Invalid_Underscore"; //$NON-NLS-1$ public static final String UNDERSCORES_IN_LITERALS_NOT_BELOW_17 = "Underscores_In_Literals_Not_Below_17"; //$NON-NLS-1$ //----------------optimized identifier managment------------------ static final char[] charArray_a = new char[] {'a'}, charArray_b = new char[] {'b'}, charArray_c = new char[] {'c'}, charArray_d = new char[] {'d'}, charArray_e = new char[] {'e'}, charArray_f = new char[] {'f'}, charArray_g = new char[] {'g'}, charArray_h = new char[] {'h'}, charArray_i = new char[] {'i'}, charArray_j = new char[] {'j'}, charArray_k = new char[] {'k'}, charArray_l = new char[] {'l'}, charArray_m = new char[] {'m'}, charArray_n = new char[] {'n'}, charArray_o = new char[] {'o'}, charArray_p = new char[] {'p'}, charArray_q = new char[] {'q'}, charArray_r = new char[] {'r'}, charArray_s = new char[] {'s'}, charArray_t = new char[] {'t'}, charArray_u = new char[] {'u'}, charArray_v = new char[] {'v'}, charArray_w = new char[] {'w'}, charArray_x = new char[] {'x'}, charArray_y = new char[] {'y'}, charArray_z = new char[] {'z'}; static final char[] initCharArray = new char[] {'\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000'}; static final int TableSize = 30, InternalTableSize = 6; //30*6 =210 entries public static final int OptimizedLength = 7; public /*static*/ final char[][][][] charArray_length = new char[OptimizedLength][TableSize][InternalTableSize][]; // support for detecting non-externalized string literals public static final char[] TAG_PREFIX= "//$NON-NLS-".toCharArray(); //$NON-NLS-1$ public static final int TAG_PREFIX_LENGTH= TAG_PREFIX.length; public static final char TAG_POSTFIX= '$'; public static final int TAG_POSTFIX_LENGTH= 1; // support for complaining on uninterned type comparisons. public static final char[] IDENTITY_COMPARISON_TAG = "//$IDENTITY-COMPARISON$".toCharArray(); //$NON-NLS-1$ public boolean [] validIdentityComparisonLines; public boolean checkUninternedIdentityComparison; private NLSTag[] nlsTags = null; protected int nlsTagsPtr; public boolean checkNonExternalizedStringLiterals; protected int lastPosition; // generic support public boolean returnOnlyGreater = false; /*static*/ { for (int i = 0; i < 6; i++) { for (int j = 0; j < TableSize; j++) { for (int k = 0; k < InternalTableSize; k++) { this.charArray_length[i][j][k] = initCharArray; } } } } /*static*/ int newEntry2 = 0, newEntry3 = 0, newEntry4 = 0, newEntry5 = 0, newEntry6 = 0; public boolean insideRecovery = false; int lookBack[] = new int[2]; // fall back to spring forward. protected int nextToken = TokenNameNotAToken; // allows for one token push back, only the most recent token can be reliably ungotten. private VanguardScanner vanguardScanner; private VanguardParser vanguardParser; ConflictedParser activeParser = null; private boolean consumingEllipsisAnnotations = false; public static final int RoundBracket = 0; public static final int SquareBracket = 1; public static final int CurlyBracket = 2; public static final int BracketKinds = 3; // extended unicode support public static final int LOW_SURROGATE_MIN_VALUE = 0xDC00; public static final int HIGH_SURROGATE_MIN_VALUE = 0xD800; public static final int HIGH_SURROGATE_MAX_VALUE = 0xDBFF; public static final int LOW_SURROGATE_MAX_VALUE = 0xDFFF; // text block support - 13 /* package */ int rawStart = -1; public Scanner() { this(false /*comment*/, false /*whitespace*/, false /*nls*/, ClassFileConstants.JDK1_3 /*sourceLevel*/, null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/); } public Scanner( boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, long complianceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive, boolean isPreviewEnabled) { this.eofPosition = Integer.MAX_VALUE; this.tokenizeComments = tokenizeComments; this.tokenizeWhiteSpace = tokenizeWhiteSpace; this.sourceLevel = sourceLevel; this.lookBack[0] = this.lookBack[1] = this.nextToken = TokenNameNotAToken; this.consumingEllipsisAnnotations = false; this.complianceLevel = complianceLevel; this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals; this.previewEnabled = isPreviewEnabled; if (taskTags != null) { int taskTagsLength = taskTags.length; int length = taskTagsLength; if (taskPriorities != null) { int taskPrioritiesLength = taskPriorities.length; if (taskPrioritiesLength != taskTagsLength) { if (taskPrioritiesLength > taskTagsLength) { System.arraycopy(taskPriorities, 0, (taskPriorities = new char[taskTagsLength][]), 0, taskTagsLength); } else { System.arraycopy(taskTags, 0, (taskTags = new char[taskPrioritiesLength][]), 0, taskPrioritiesLength); length = taskPrioritiesLength; } } int[] initialIndexes = new int[length]; for (int i = 0; i < length; i++) { initialIndexes[i] = i; } Util.reverseQuickSort(taskTags, 0, length - 1, initialIndexes); char[][] temp = new char[length][]; for (int i = 0; i < length; i++) { temp[i] = taskPriorities[initialIndexes[i]]; } this.taskPriorities = temp; } else { Util.reverseQuickSort(taskTags, 0, length - 1); } this.taskTags = taskTags; this.isTaskCaseSensitive = isTaskCaseSensitive; } } public Scanner( boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive, boolean isPreviewEnabled) { this( tokenizeComments, tokenizeWhiteSpace, checkNonExternalizedStringLiterals, sourceLevel, sourceLevel, taskTags, taskPriorities, isTaskCaseSensitive, isPreviewEnabled); } public Scanner( boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive) { this( tokenizeComments, tokenizeWhiteSpace, checkNonExternalizedStringLiterals, sourceLevel, sourceLevel, taskTags, taskPriorities, isTaskCaseSensitive, false); } public final boolean atEnd() { // This code is not relevant if source is // Only a part of the real stream input return this.eofPosition <= this.currentPosition; } // chech presence of task: tags // TODO (frederic) see if we need to take unicode characters into account... public void checkTaskTag(int commentStart, int commentEnd) throws InvalidInputException { char[] src = this.source; // only look for newer task: tags if (this.foundTaskCount > 0 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) { return; } int foundTaskIndex = this.foundTaskCount; char previous = src[commentStart+1]; // should be '*' or '/' for ( int i = commentStart + 2; i < commentEnd && i < this.eofPosition; i++) { char[] tag = null; char[] priority = null; // check for tag occurrence only if not ambiguous with javadoc tag if (previous != '@') { nextTag : for (int itag = 0; itag < this.taskTags.length; itag++) { tag = this.taskTags[itag]; int tagLength = tag.length; if (tagLength == 0) continue nextTag; // ensure tag is not leaded with letter if tag starts with a letter if (ScannerHelper.isJavaIdentifierStart(this.complianceLevel, tag[0])) { if (ScannerHelper.isJavaIdentifierPart(this.complianceLevel, previous)) { continue nextTag; } } for (int t = 0; t < tagLength; t++) { char sc, tc; int x = i+t; if (x >= this.eofPosition || x >= commentEnd) continue nextTag; // case sensitive check if ((sc = src[i + t]) != (tc = tag[t])) { // case insensitive check if (this.isTaskCaseSensitive || (ScannerHelper.toLowerCase(sc) != ScannerHelper.toLowerCase(tc))) { continue nextTag; } } } // ensure tag is not followed with letter if tag finishes with a letter if (i+tagLength < commentEnd && ScannerHelper.isJavaIdentifierPart(this.complianceLevel, src[i+tagLength-1])) { if (ScannerHelper.isJavaIdentifierPart(this.complianceLevel, src[i + tagLength])) continue nextTag; } if (this.foundTaskTags == null) { this.foundTaskTags = new char[5][]; this.foundTaskMessages = new char[5][]; this.foundTaskPriorities = new char[5][]; this.foundTaskPositions = new int[5][]; } else if (this.foundTaskCount == this.foundTaskTags.length) { System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount); System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount); System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount); System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0, this.foundTaskCount); } priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null; this.foundTaskTags[this.foundTaskCount] = tag; this.foundTaskPriorities[this.foundTaskCount] = priority; this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 }; this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR; this.foundTaskCount++; i += tagLength - 1; // will be incremented when looping break nextTag; } } previous = src[i]; } boolean containsEmptyTask = false; for (int i = foundTaskIndex; i < this.foundTaskCount; i++) { // retrieve message start and end positions int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length; int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1 : commentEnd - 1; // at most beginning of next task if (max_value < msgStart) { max_value = msgStart; // would only occur if tag is before EOF. } int end = -1; char c; for (int j = msgStart; j < max_value; j++) { if ((c = src[j]) == '\n' || c == '\r') { end = j - 1; break; } } if (end == -1) { for (int j = max_value; j > msgStart; j--) { if ((c = src[j]) == '*') { end = j - 1; break; } } if (end == -1) end = max_value; } if (msgStart == end) { // if the description is empty, we might want to see if two tags are not sharing the same message // see https://bugs.eclipse.org/bugs/show_bug.cgi?id=110797 containsEmptyTask = true; continue; } // trim the message // we don't trim the beginning of the message to be able to show it after the task tag while (CharOperation.isWhitespace(src[end]) && msgStart <= end) end--; // update the end position of the task this.foundTaskPositions[i][1] = end; // get the message source final int messageLength = end - msgStart + 1; char[] message = new char[messageLength]; System.arraycopy(src, msgStart, message, 0, messageLength); this.foundTaskMessages[i] = message; } if (containsEmptyTask) { for (int i = foundTaskIndex, max = this.foundTaskCount; i < max; i++) { if (this.foundTaskMessages[i].length == 0) { loop: for (int j = i + 1; j < max; j++) { if (this.foundTaskMessages[j].length != 0) { this.foundTaskMessages[i] = this.foundTaskMessages[j]; this.foundTaskPositions[i][1] = this.foundTaskPositions[j][1]; break loop; } } } } } } public char[] getCurrentIdentifierSource() { //return the token REAL source (aka unicodes are precomputed) if (this.withoutUnicodePtr != 0) { //0 is used as a fast test flag so the real first char is in position 1 char[] result = new char[this.withoutUnicodePtr]; System.arraycopy( this.withoutUnicodeBuffer, 1, result, 0, this.withoutUnicodePtr); return result; } int length = this.currentPosition - this.startPosition; if (length == this.eofPosition) return this.source; switch (length) { // see OptimizedLength case 1 : return optimizedCurrentTokenSource1(); case 2 : return optimizedCurrentTokenSource2(); case 3 : return optimizedCurrentTokenSource3(); case 4 : return optimizedCurrentTokenSource4(); case 5 : return optimizedCurrentTokenSource5(); case 6 : return optimizedCurrentTokenSource6(); } char[] result = new char[length]; System.arraycopy(this.source, this.startPosition, result, 0, length); return result; } public int getCurrentTokenEndPosition(){ return this.currentPosition - 1; } public char[] getCurrentTokenSource() { // Return the token REAL source (aka unicodes are precomputed) char[] result; if (this.withoutUnicodePtr != 0) // 0 is used as a fast test flag so the real first char is in position 1 System.arraycopy( this.withoutUnicodeBuffer, 1, result = new char[this.withoutUnicodePtr], 0, this.withoutUnicodePtr); else { int length; System.arraycopy( this.source, this.startPosition, result = new char[length = this.currentPosition - this.startPosition], 0, length); } return result; } public final String getCurrentTokenString() { // Return current token as a string if (this.withoutUnicodePtr != 0) { // 0 is used as a fast test flag so the real first char is in position 1 return new String( this.withoutUnicodeBuffer, 1, this.withoutUnicodePtr); } return new String( this.source, this.startPosition, this.currentPosition - this.startPosition); } public char[] getCurrentTokenSourceString() { //return the token REAL source (aka unicodes are precomputed). //REMOVE the two " that are at the beginning and the end. char[] result; if (this.withoutUnicodePtr != 0) //0 is used as a fast test flag so the real first char is in position 1 System.arraycopy(this.withoutUnicodeBuffer, 2, //2 is 1 (real start) + 1 (to jump over the ") result = new char[this.withoutUnicodePtr - 2], 0, this.withoutUnicodePtr - 2); else { int length; System.arraycopy( this.source, this.startPosition + 1, result = new char[length = this.currentPosition - this.startPosition - 2], 0, length); } return result; } protected final boolean scanForTextBlockBeginning() { if (this.activeParser != null && !this.activeParser.isParsingJava13()) { return false; } try { // Don't change the position and current character unless we are certain // to be dealing with a text block. For producing all errors like before // in case of a valid """ but missing \r or \n, just return false and not // throw any error. int temp = this.currentPosition; if ((this.source[temp++] == '\"' && this.source[temp++] == '\"')) { char c = this.source[temp++]; while (ScannerHelper.isWhitespace(c)) { switch (c) { case 10 : /* \ u000a: LINE FEED */ case 13 : /* \ u000d: CARRIAGE RETURN */ this.currentCharacter = c; this.currentPosition = temp; return true; default: break; } c = this.source[temp++]; } } } catch(IndexOutOfBoundsException e) { //let it return false; } return false; } protected final boolean scanForTextBlockClose() throws InvalidInputException { try { if (this.source[this.currentPosition] == '\"' && this.source[this.currentPosition + 1] == '\"') { return true; } } catch(IndexOutOfBoundsException e) { //let it return false; } return false; } public char[] getCurrentTextBlock() { // 1. Normalize, i.e. convert all CR CRLF to LF char[] all; if (this.withoutUnicodePtr != 0) { all = CharOperation.subarray(this.withoutUnicodeBuffer, this.rawStart + 1, this.withoutUnicodePtr + 1 ); } else { all = CharOperation.subarray(this.source, this.startPosition + this.rawStart, this.currentPosition - 3); if (all == null) { all = new char[0]; } } // 2. Split into lines. Consider both \n and \r as line separators char[][] lines = CharOperation.splitOn('\n', all); int size = lines.length; List<char[]> list = new ArrayList<>(lines.length); for(int i = 0; i < lines.length; i++) { char[] line = lines[i]; if (i + 1 == size && line.length == 0) { list.add(line); break; } char[][] sub = CharOperation.splitOn('\r', line); for (char[] cs : sub) { if (cs.length > 0) { list.add(cs); } } } size = list.size(); lines = list.toArray(new char[size][]); // 3. Handle incidental white space // 3.1. Split into lines and identify determining lines int prefix = -1; for(int i = 0; i < size; i++) { char[] line = lines[i]; boolean blank = true; int whitespaces = 0; for (char c : line) { if (blank) { if (ScannerHelper.isWhitespace(c)) { whitespaces++; } else { blank = false; } } } if (!blank || (i+1 == size)) { if (prefix < 0 || whitespaces < prefix) { prefix = whitespaces; } } } // 3.2. Remove the common white space prefix // 4. Handle escape sequences (already done while processing if (prefix == -1) prefix = 0; char[] result = new char[0]; for(int i = 0; i < lines.length; i++) { char[] l = lines[i]; // Remove the common prefix from each line // And remove all trailing whitespace // Finally append the \n at the end of the line (except the last line) int length = l.length; int trail = length - 1; for(int j = trail; j>0; j--) { if (!ScannerHelper.isWhitespace(l[j])) { trail = j; break; } } int newSize = (length == 0 || prefix > trail) ? 0 : (trail - prefix + 1); char[] nl; if (i >= (size - 1)) { if (trail <= 0 || newSize == 0) continue; nl = new char[newSize]; System.arraycopy(l, prefix, nl, 0, newSize); } else { newSize += 1; nl = new char[newSize]; nl[newSize - 1] = '\n'; if (newSize > 1) System.arraycopy(l, prefix, nl, 0, newSize - 1); } result = CharOperation.concat(result, nl); } // get rid of all the cached values this.rawStart = -1; return result; } public final String getCurrentStringLiteral() { //return the token REAL source (aka unicodes are precomputed). //REMOVE the two " that are at the beginning and the end. if (this.withoutUnicodePtr != 0) //0 is used as a fast test flag so the real first char is in position 1 //2 is 1 (real start) + 1 (to jump over the ") return new String(this.withoutUnicodeBuffer, 2, this.withoutUnicodePtr - 2); else { return new String(this.source, this.startPosition + 1, this.currentPosition - this.startPosition - 2); } } public final char[] getRawTokenSource() { int length = this.currentPosition - this.startPosition; char[] tokenSource = new char[length]; System.arraycopy(this.source, this.startPosition, tokenSource, 0, length); return tokenSource; } public final char[] getRawTokenSourceEnd() { int length = this.eofPosition - this.currentPosition - 1; char[] sourceEnd = new char[length]; System.arraycopy(this.source, this.currentPosition, sourceEnd, 0, length); return sourceEnd; } public int getCurrentTokenStartPosition(){ return this.startPosition; } /* * Search the source position corresponding to the end of a given line number * * Line numbers are 1-based, and relative to the scanner initialPosition. * Character positions are 0-based. * * In case the given line number is inconsistent, answers -1. */ public final int getLineEnd(int lineNumber) { if (this.lineEnds == null || this.linePtr == -1) return -1; if (lineNumber > this.lineEnds.length+1) return -1; if (lineNumber <= 0) return -1; if (lineNumber == this.lineEnds.length + 1) return this.eofPosition; return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line } public final int[] getLineEnds() { //return a bounded copy of this.lineEnds if (this.linePtr == -1) { return EMPTY_LINE_ENDS; } int[] copy; System.arraycopy(this.lineEnds, 0, copy = new int[this.linePtr + 1], 0, this.linePtr + 1); return copy; }
Search the source position corresponding to the beginning of a given line number Line numbers are 1-based, and relative to the scanner initialPosition. Character positions are 0-based. e.g. getLineStart(1) --> 0 indicates that the first line starts at character 0. In case the given line number is inconsistent, answers -1.
Params:
  • lineNumber – int
Returns:int
/** * Search the source position corresponding to the beginning of a given line number * * Line numbers are 1-based, and relative to the scanner initialPosition. * Character positions are 0-based. * * e.g. getLineStart(1) --> 0 indicates that the first line starts at character 0. * * In case the given line number is inconsistent, answers -1. * * @param lineNumber int * @return int */
public final int getLineStart(int lineNumber) { if (this.lineEnds == null || this.linePtr == -1) return -1; if (lineNumber > this.lineEnds.length + 1) return -1; if (lineNumber <= 0) return -1; if (lineNumber == 1) return this.initialPosition; return this.lineEnds[lineNumber-2]+1; // next line start one character behind the lineEnd of the previous line } public final int getNextChar() { try { if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { this.unicodeAsBackSlash = false; if (this.withoutUnicodePtr != 0) { unicodeStore(); } } return this.currentCharacter; } catch(IndexOutOfBoundsException | InvalidInputException e) { return -1; } } public final int getNextCharWithBoundChecks() { if (this.currentPosition >= this.eofPosition) { return -1; } this.currentCharacter = this.source[this.currentPosition++]; if (this.currentPosition >= this.eofPosition) { this.unicodeAsBackSlash = false; if (this.withoutUnicodePtr != 0) { unicodeStore(); } return this.currentCharacter; } if (this.currentCharacter == '\\' && this.source[this.currentPosition] == 'u') { try { getNextUnicodeChar(); } catch (InvalidInputException e) { return -1; } } else { this.unicodeAsBackSlash = false; if (this.withoutUnicodePtr != 0) { unicodeStore(); } } return this.currentCharacter; } public final boolean getNextChar(char testedChar) { //BOOLEAN //handle the case of unicode. //when a unicode appears then we must use a buffer that holds char internal values //At the end of this method currentCharacter holds the new visited char //and currentPosition points right next after it //Both previous lines are true if the currentCharacter is == to the testedChar //On false, no side effect has occured. //ALL getNextChar.... ARE OPTIMIZED COPIES if (this.currentPosition >= this.eofPosition) { // handle the obvious case upfront this.unicodeAsBackSlash = false; return false; } int temp = this.currentPosition; try { if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); if (this.currentCharacter != testedChar) { this.currentPosition = temp; this.withoutUnicodePtr--; return false; } return true; } //-------------end unicode traitement-------------- else { if (this.currentCharacter != testedChar) { this.currentPosition = temp; return false; } this.unicodeAsBackSlash = false; if (this.withoutUnicodePtr != 0) unicodeStore(); return true; } } catch(IndexOutOfBoundsException | InvalidInputException e) { this.unicodeAsBackSlash = false; this.currentPosition = temp; return false; } } public final int getNextChar(char testedChar1, char testedChar2) { //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others //test can be done with (x==0) for the first and (x>0) for the second //handle the case of unicode. //when a unicode appears then we must use a buffer that holds char internal values //At the end of this method currentCharacter holds the new visited char //and currentPosition points right next after it //Both previous lines are true if the currentCharacter is == to the testedChar1/2 //On false, no side effect has occured. //ALL getNextChar.... ARE OPTIMIZED COPIES if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront return -1; int temp = this.currentPosition; try { int result; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); if (this.currentCharacter == testedChar1) { result = 0; } else if (this.currentCharacter == testedChar2) { result = 1; } else { this.currentPosition = temp; this.withoutUnicodePtr--; result = -1; } return result; } else { if (this.currentCharacter == testedChar1) { result = 0; } else if (this.currentCharacter == testedChar2) { result = 1; } else { this.currentPosition = temp; return -1; } if (this.withoutUnicodePtr != 0) unicodeStore(); return result; } } catch(IndexOutOfBoundsException | InvalidInputException e) { this.currentPosition = temp; return -1; } } /* * This method consumes digits as well as underscores if underscores are located between digits * @throws InvalidInputException if underscores are not located between digits or if underscores are used in source < 1.7 */ private final void consumeDigits(int radix) throws InvalidInputException { consumeDigits(radix, false); } /* * This method consumes digits as well as underscores if underscores are located between digits * @throws InvalidInputException if underscores are not located between digits or if underscores are used in source < 1.7 */ private final void consumeDigits(int radix, boolean expectingDigitFirst) throws InvalidInputException { final int USING_UNDERSCORE = 1; final int INVALID_POSITION = 2; switch(consumeDigits0(radix, USING_UNDERSCORE, INVALID_POSITION, expectingDigitFirst)) { case USING_UNDERSCORE : if (this.sourceLevel < ClassFileConstants.JDK1_7) { throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17); } break; case INVALID_POSITION : if (this.sourceLevel < ClassFileConstants.JDK1_7) { throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17); } throw new InvalidInputException(INVALID_UNDERSCORE); } } private final int consumeDigits0(int radix, int usingUnderscore, int invalidPosition, boolean expectingDigitFirst) throws InvalidInputException { int kind = 0; if (getNextChar('_')) { if (expectingDigitFirst) { return invalidPosition; } kind = usingUnderscore; while (getNextChar('_')) {/*empty */} } if (getNextCharAsDigit(radix)) { // continue to read digits or underscore while (getNextCharAsDigit(radix)) {/*empty */} int kind2 = consumeDigits0(radix, usingUnderscore, invalidPosition, false); if (kind2 == 0) { return kind; } return kind2; } if (kind == usingUnderscore) return invalidPosition; return kind; } public final boolean getNextCharAsDigit() throws InvalidInputException { //BOOLEAN //handle the case of unicode. //when a unicode appears then we must use a buffer that holds char internal values //At the end of this method currentCharacter holds the new visited char //and currentPosition points right next after it //Both previous lines are true if the currentCharacter is a digit //On false, no side effect has occured. //ALL getNextChar.... ARE OPTIMIZED COPIES if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront return false; int temp = this.currentPosition; try { if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); if (!ScannerHelper.isDigit(this.currentCharacter)) { this.currentPosition = temp; this.withoutUnicodePtr--; return false; } return true; } else { if (!ScannerHelper.isDigit(this.currentCharacter)) { this.currentPosition = temp; return false; } if (this.withoutUnicodePtr != 0) unicodeStore(); return true; } } catch(IndexOutOfBoundsException | InvalidInputException e) { this.currentPosition = temp; return false; } } public final boolean getNextCharAsDigit(int radix) { //BOOLEAN //handle the case of unicode. //when a unicode appears then we must use a buffer that holds char internal values //At the end of this method currentCharacter holds the new visited char //and currentPosition points right next after it //Both previous lines are true if the currentCharacter is a digit base on radix //On false, no side effect has occured. //ALL getNextChar.... ARE OPTIMIZED COPIES if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront return false; int temp = this.currentPosition; try { if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); if (ScannerHelper.digit(this.currentCharacter, radix) == -1) { this.currentPosition = temp; this.withoutUnicodePtr--; return false; } return true; } else { if (ScannerHelper.digit(this.currentCharacter, radix) == -1) { this.currentPosition = temp; return false; } if (this.withoutUnicodePtr != 0) unicodeStore(); return true; } } catch(IndexOutOfBoundsException | InvalidInputException e) { this.currentPosition = temp; return false; } } public boolean getNextCharAsJavaIdentifierPartWithBoundCheck() { //BOOLEAN //handle the case of unicode. //when a unicode appears then we must use a buffer that holds char internal values //At the end of this method currentCharacter holds the new visited char //and currentPosition points right next after it //Both previous lines are true if the currentCharacter is a JavaIdentifierPart //On false, no side effect has occured. //ALL getNextChar.... ARE OPTIMIZED COPIES int pos = this.currentPosition; if (pos >= this.eofPosition) // handle the obvious case upfront return false; int temp2 = this.withoutUnicodePtr; try { boolean unicode = false; this.currentCharacter = this.source[this.currentPosition++]; if (this.currentPosition < this.eofPosition) { if (this.currentCharacter == '\\' && this.source[this.currentPosition] == 'u') { getNextUnicodeChar(); unicode = true; } } char c = this.currentCharacter; boolean isJavaIdentifierPart = false; if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) { if (this.complianceLevel < ClassFileConstants.JDK1_5) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } // Unicode 4 detection char low = (char) getNextCharWithBoundChecks(); if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) { // illegal low surrogate this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c, low); } else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } else { isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c); } if (unicode) { if (!isJavaIdentifierPart) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } return true; } else { if (!isJavaIdentifierPart) { this.currentPosition = pos; return false; } if (this.withoutUnicodePtr != 0) unicodeStore(); return true; } } catch(InvalidInputException e) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } } public boolean getNextCharAsJavaIdentifierPart() { //BOOLEAN //handle the case of unicode. //when a unicode appears then we must use a buffer that holds char internal values //At the end of this method currentCharacter holds the new visited char //and currentPosition points right next after it //Both previous lines are true if the currentCharacter is a JavaIdentifierPart //On false, no side effect has occured. //ALL getNextChar.... ARE OPTIMIZED COPIES int pos; if ((pos = this.currentPosition) >= this.eofPosition) // handle the obvious case upfront return false; int temp2 = this.withoutUnicodePtr; try { boolean unicode = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); unicode = true; } char c = this.currentCharacter; boolean isJavaIdentifierPart = false; if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) { if (this.complianceLevel < ClassFileConstants.JDK1_5) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } // Unicode 4 detection char low = (char) getNextChar(); if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) { // illegal low surrogate this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c, low); } else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } else { isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c); } if (unicode) { if (!isJavaIdentifierPart) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } return true; } else { if (!isJavaIdentifierPart) { this.currentPosition = pos; return false; } if (this.withoutUnicodePtr != 0) unicodeStore(); return true; } } catch(IndexOutOfBoundsException | InvalidInputException e) { this.currentPosition = pos; this.withoutUnicodePtr = temp2; return false; } } /* * External API in JavaConventions. * This is used to optimize the case where the scanner is used to scan a single identifier. * In this case, the AIOOBE is slower to handle than a bound check */ public int scanIdentifier() throws InvalidInputException { int whiteStart = 0; while (true) { //loop for jumping over comments this.withoutUnicodePtr = 0; //start with a new token (even comment written with unicode ) // ---------Consume white space and handles startPosition--------- whiteStart = this.currentPosition; boolean isWhiteSpace, hasWhiteSpaces = false; int offset; int unicodePtr; boolean checkIfUnicode = false; do { unicodePtr = this.withoutUnicodePtr; offset = this.currentPosition; this.startPosition = this.currentPosition; if (this.currentPosition < this.eofPosition) { this.currentCharacter = this.source[this.currentPosition++]; checkIfUnicode = this.currentPosition < this.eofPosition && this.currentCharacter == '\\' && this.source[this.currentPosition] == 'u'; } else if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) { // reposition scanner in case we are interested by spaces as tokens this.currentPosition--; this.startPosition = whiteStart; return TokenNameWHITESPACE; } else { return TokenNameEOF; } if (checkIfUnicode) { isWhiteSpace = jumpOverUnicodeWhiteSpace(); offset = this.currentPosition - offset; } else { offset = this.currentPosition - offset; // inline version of: //isWhiteSpace = // (this.currentCharacter == ' ') || ScannerHelper.isWhitespace(this.currentCharacter); switch (this.currentCharacter) { case 10 : /* \ u000a: LINE FEED */ case 12 : /* \ u000c: FORM FEED */ case 13 : /* \ u000d: CARRIAGE RETURN */ case 32 : /* \ u0020: SPACE */ case 9 : /* \ u0009: HORIZONTAL TABULATION */ isWhiteSpace = true; break; default : isWhiteSpace = false; } } if (isWhiteSpace) { hasWhiteSpaces = true; } } while (isWhiteSpace); if (hasWhiteSpaces) { if (this.tokenizeWhiteSpace) { // reposition scanner in case we are interested by spaces as tokens this.currentPosition-=offset; this.startPosition = whiteStart; if (checkIfUnicode) { this.withoutUnicodePtr = unicodePtr; } return TokenNameWHITESPACE; } else if (checkIfUnicode) { this.withoutUnicodePtr = 0; unicodeStore(); } else { this.withoutUnicodePtr = 0; } } char c = this.currentCharacter; if (c < ScannerHelper.MAX_OBVIOUS) { if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) { return scanIdentifierOrKeywordWithBoundCheck(); } return TokenNameERROR; } boolean isJavaIdStart; if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) { if (this.complianceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } // Unicode 4 detection char low = (char) getNextCharWithBoundChecks(); if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) { // illegal low surrogate throw new InvalidInputException(INVALID_LOW_SURROGATE); } isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c, low); } else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) { if (this.complianceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } throw new InvalidInputException(INVALID_HIGH_SURROGATE); } else { // optimized case already checked isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c); } if (isJavaIdStart) return scanIdentifierOrKeywordWithBoundCheck(); return TokenNameERROR; } } public void ungetToken(int unambiguousToken) { if (this.nextToken != TokenNameNotAToken) { throw new ArrayIndexOutOfBoundsException("Single cell array overflow"); //$NON-NLS-1$ } this.nextToken = unambiguousToken; } private void updateCase(int token) { if (token == TokenNamecase) { this.inCase = true; this.breakPreviewAllowed = true; } if (token == TokenNameCOLON || token == TokenNameARROW) this.inCase = false; } public int getNextToken() throws InvalidInputException { int token; if (this.nextToken != TokenNameNotAToken) { token = this.nextToken; this.nextToken = TokenNameNotAToken; return token; // presumed to be unambiguous. } if (this.scanContext == null) { // init lazily, since isInModuleDeclaration needs the parser to be known this.scanContext = isInModuleDeclaration() ? ScanContext.EXPECTING_KEYWORD : ScanContext.INACTIVE; } token = getNextToken0(); if (areRestrictedModuleKeywordsActive()) { if (isRestrictedKeyword(token)) token = disambiguatedRestrictedKeyword(token); updateScanContext(token); } if (this.activeParser == null) { // anybody interested in the grammatical structure of the program should have registered. return token; } if (token == TokenNameLPAREN || token == TokenNameLESS || token == TokenNameAT || token == TokenNameARROW) { token = disambiguatedToken(token); } else if (token == TokenNameELLIPSIS) { this.consumingEllipsisAnnotations = false; } this.lookBack[0] = this.lookBack[1]; this.lookBack[1] = token; updateCase(token); return token; } protected int getNextToken0() throws InvalidInputException { this.wasAcr = false; if (this.diet) { jumpOverMethodBody(); this.diet = false; return this.currentPosition > this.eofPosition ? TokenNameEOF : TokenNameRBRACE; } int whiteStart = 0; try { while (true) { //loop for jumping over comments this.withoutUnicodePtr = 0; //start with a new token (even comment written with unicode ) // ---------Consume white space and handles startPosition--------- whiteStart = this.currentPosition; boolean isWhiteSpace, hasWhiteSpaces = false; int offset; int unicodePtr; boolean checkIfUnicode = false; do { unicodePtr = this.withoutUnicodePtr; offset = this.currentPosition; this.startPosition = this.currentPosition; try { checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u'); } catch(IndexOutOfBoundsException e) { if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) { // reposition scanner in case we are interested by spaces as tokens this.currentPosition--; this.startPosition = whiteStart; return TokenNameWHITESPACE; } if (this.currentPosition > this.eofPosition) return TokenNameEOF; } if (this.currentPosition > this.eofPosition) { if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) { this.currentPosition--; // reposition scanner in case we are interested by spaces as tokens this.startPosition = whiteStart; return TokenNameWHITESPACE; } return TokenNameEOF; } if (checkIfUnicode) { isWhiteSpace = jumpOverUnicodeWhiteSpace(); offset = this.currentPosition - offset; } else { offset = this.currentPosition - offset; if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) { if (this.recordLineSeparator) { pushLineSeparator(); } } // inline version of: //isWhiteSpace = // (this.currentCharacter == ' ') || ScannerHelper.isWhitespace(this.currentCharacter); switch (this.currentCharacter) { case 10 : /* \ u000a: LINE FEED */ case 12 : /* \ u000c: FORM FEED */ case 13 : /* \ u000d: CARRIAGE RETURN */ case 32 : /* \ u0020: SPACE */ case 9 : /* \ u0009: HORIZONTAL TABULATION */ isWhiteSpace = true; break; default : isWhiteSpace = false; } } if (isWhiteSpace) { hasWhiteSpaces = true; } } while (isWhiteSpace); if (hasWhiteSpaces) { if (this.tokenizeWhiteSpace) { // reposition scanner in case we are interested by spaces as tokens this.currentPosition-=offset; this.startPosition = whiteStart; if (checkIfUnicode) { this.withoutUnicodePtr = unicodePtr; } return TokenNameWHITESPACE; } else if (checkIfUnicode) { this.withoutUnicodePtr = 0; unicodeStore(); } else { this.withoutUnicodePtr = 0; } } // ---------Identify the next token------------- switch (this.currentCharacter) { case '@' : /* if (this.sourceLevel >= ClassFileConstants.JDK1_5) { return TokenNameAT; } else { return TokenNameERROR; }*/ return TokenNameAT; case '(' : return TokenNameLPAREN; case ')' : return TokenNameRPAREN; case '{' : return TokenNameLBRACE; case '}' : return TokenNameRBRACE; case '[' : return TokenNameLBRACKET; case ']' : return TokenNameRBRACKET; case ';' : return TokenNameSEMICOLON; case ',' : return TokenNameCOMMA; case '.' : if (getNextCharAsDigit()) { return scanNumber(true); } int temp = this.currentPosition; if (getNextChar('.')) { if (getNextChar('.')) { return TokenNameELLIPSIS; } else { this.currentPosition = temp; return TokenNameDOT; } } else { this.currentPosition = temp; return TokenNameDOT; } case '+' : { int test; if ((test = getNextChar('+', '=')) == 0) return TokenNamePLUS_PLUS; if (test > 0) return TokenNamePLUS_EQUAL; return TokenNamePLUS; } case '-' : { int test; if ((test = getNextChar('-', '=')) == 0) return TokenNameMINUS_MINUS; if (test > 0) return TokenNameMINUS_EQUAL; if (getNextChar('>')) return TokenNameARROW; return TokenNameMINUS; } case '~' : return TokenNameTWIDDLE; case '!' : if (getNextChar('=')) return TokenNameNOT_EQUAL; return TokenNameNOT; case '*' : if (getNextChar('=')) return TokenNameMULTIPLY_EQUAL; return TokenNameMULTIPLY; case '%' : if (getNextChar('=')) return TokenNameREMAINDER_EQUAL; return TokenNameREMAINDER; case '<' : { int test; if ((test = getNextChar('=', '<')) == 0) return TokenNameLESS_EQUAL; if (test > 0) { if (getNextChar('=')) return TokenNameLEFT_SHIFT_EQUAL; return TokenNameLEFT_SHIFT; } return TokenNameLESS; } case '>' : { int test; if (this.returnOnlyGreater) { return TokenNameGREATER; } if ((test = getNextChar('=', '>')) == 0) return TokenNameGREATER_EQUAL; if (test > 0) { if ((test = getNextChar('=', '>')) == 0) return TokenNameRIGHT_SHIFT_EQUAL; if (test > 0) { if (getNextChar('=')) return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL; return TokenNameUNSIGNED_RIGHT_SHIFT; } return TokenNameRIGHT_SHIFT; } return TokenNameGREATER; } case '=' : if (getNextChar('=')) return TokenNameEQUAL_EQUAL; return TokenNameEQUAL; case '&' : { int test; if ((test = getNextChar('&', '=')) == 0) return TokenNameAND_AND; if (test > 0) return TokenNameAND_EQUAL; return TokenNameAND; } case '|' : { int test; if ((test = getNextChar('|', '=')) == 0) return TokenNameOR_OR; if (test > 0) return TokenNameOR_EQUAL; return TokenNameOR; } case '^' : if (getNextChar('=')) return TokenNameXOR_EQUAL; return TokenNameXOR; case '?' : return TokenNameQUESTION; case ':' : if (getNextChar(':')) return TokenNameCOLON_COLON; ++this.yieldColons; return TokenNameCOLON; case '\'' : { int test; if ((test = getNextChar('\n', '\r')) == 0) { throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); } if (test > 0) { // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed for (int lookAhead = 0; lookAhead < 3; lookAhead++) { if (this.currentPosition + lookAhead == this.eofPosition) break; if (this.source[this.currentPosition + lookAhead] == '\n') break; if (this.source[this.currentPosition + lookAhead] == '\'') { this.currentPosition += lookAhead + 1; break; } } throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); } } if (getNextChar('\'')) { // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed for (int lookAhead = 0; lookAhead < 3; lookAhead++) { if (this.currentPosition + lookAhead == this.eofPosition) break; if (this.source[this.currentPosition + lookAhead] == '\n') break; if (this.source[this.currentPosition + lookAhead] == '\'') { this.currentPosition += lookAhead + 1; break; } } throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); } if (getNextChar('\\')) { if (this.unicodeAsBackSlash) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } else { this.currentCharacter = this.source[this.currentPosition++]; } scanEscapeCharacter(); } else { // consume next character this.unicodeAsBackSlash = false; checkIfUnicode = false; try { checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u'); } catch(IndexOutOfBoundsException e) { this.currentPosition--; throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); } if (checkIfUnicode) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } if (getNextChar('\'')) return TokenNameCharacterLiteral; // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed for (int lookAhead = 0; lookAhead < 20; lookAhead++) { if (this.currentPosition + lookAhead == this.eofPosition) break; if (this.source[this.currentPosition + lookAhead] == '\n') break; if (this.source[this.currentPosition + lookAhead] == '\'') { this.currentPosition += lookAhead + 1; break; } } throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); case '"' : boolean isTextBlock = false; int lastQuotePos = 0; try { // consume next character this.unicodeAsBackSlash = false; boolean isUnicode = false; isTextBlock = scanForTextBlockBeginning(); if (!isTextBlock) { if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } this.rawStart = this.currentPosition - this.startPosition; int terminators = 0; while (this.currentPosition <= this.eofPosition) { if (this.currentCharacter == '"') { if (!isTextBlock) { return TerminalTokens.TokenNameStringLiteral; } lastQuotePos = this.currentPosition; // look for text block delimiter if (scanForTextBlockClose()) { // Account for just the snippet being passed around // If already at the EOF, bail out. if (this.currentPosition + 2 < this.source.length && this.source[this.currentPosition + 2] == '"') { terminators++; if (terminators > 2) throw new InvalidInputException(UNTERMINATED_TEXT_BLOCK); } else { this.currentPosition += 2; return TerminalTokens.TokenNameTextBlock; } } if (this.withoutUnicodePtr != 0) { unicodeStore(); } } else { terminators = 0; } if (!isTextBlock && (this.currentCharacter == '\n' || this.currentCharacter == '\r')) { // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed if (isUnicode) { int start = this.currentPosition; for (int lookAhead = 0; lookAhead < 50; lookAhead++) { if (this.currentPosition >= this.eofPosition) { this.currentPosition = start; break; } if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { isUnicode = true; getNextUnicodeChar(); } else { isUnicode = false; } if (!isUnicode && this.currentCharacter == '\n') { this.currentPosition--; // set current position on new line character break; } if (this.currentCharacter == '\"') { throw new InvalidInputException(INVALID_CHAR_IN_STRING); } } } else { this.currentPosition--; // set current position on new line character } throw new InvalidInputException(INVALID_CHAR_IN_STRING); } if (this.currentCharacter == '\\') { if (this.unicodeAsBackSlash) { this.withoutUnicodePtr--; // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; this.withoutUnicodePtr--; } else { isUnicode = false; } } else { if (this.withoutUnicodePtr == 0) { unicodeInitializeBuffer(this.currentPosition - this.startPosition); } this.withoutUnicodePtr --; this.currentCharacter = this.source[this.currentPosition++]; } // we need to compute the escape character in a separate buffer scanEscapeCharacter(); if (this.withoutUnicodePtr != 0) { unicodeStore(); } } // consume next character if (this.currentPosition >= this.eofPosition) { break; } this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; } else { isUnicode = false; if (isTextBlock && this.currentCharacter == '"') continue; if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } if (isTextBlock) { if (lastQuotePos > 0) this.currentPosition = lastQuotePos; this.currentPosition = (lastQuotePos > 0) ? lastQuotePos : this.startPosition + this.rawStart; throw new InvalidInputException(UNTERMINATED_TEXT_BLOCK); } else { throw new InvalidInputException(UNTERMINATED_STRING); } } catch (IndexOutOfBoundsException e) { if (isTextBlock) { this.currentPosition = (lastQuotePos > 0) ? lastQuotePos : this.startPosition + this.rawStart; throw new InvalidInputException(UNTERMINATED_TEXT_BLOCK); } else { this.currentPosition--; throw new InvalidInputException(UNTERMINATED_STRING); } } catch (InvalidInputException e) { if (e.getMessage().equals(INVALID_ESCAPE)) { // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed for (int lookAhead = 0; lookAhead < 50; lookAhead++) { if (this.currentPosition + lookAhead == this.eofPosition) break; if (this.source[this.currentPosition + lookAhead] == '\n') break; if (this.source[this.currentPosition + lookAhead] == '\"') { this.currentPosition += lookAhead + 1; break; } } } throw e; // rethrow } case '/' : if (!this.skipComments) { int test = getNextChar('/', '*'); if (test == 0) { //line comment this.lastCommentLinePosition = this.currentPosition; try { //get the next char if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; } //jump over the \\ boolean isUnicode = false; while (this.currentCharacter != '\r' && this.currentCharacter != '\n') { if (this.currentPosition >= this.eofPosition) { this.lastCommentLinePosition = this.currentPosition; this.currentPosition ++; // this avoids duplicating the code in the catch(IndexOutOfBoundsException e) throw new IndexOutOfBoundsException(); } this.lastCommentLinePosition = this.currentPosition; //get the next char isUnicode = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; } //jump over the \\ } /* * We need to completely consume the line break */ if (this.currentCharacter == '\r' && this.eofPosition > this.currentPosition) { if (this.source[this.currentPosition] == '\n') { this.currentPosition++; this.currentCharacter = '\n'; } else if ((this.source[this.currentPosition] == '\\') && (this.source[this.currentPosition + 1] == 'u')) { getNextUnicodeChar(); isUnicode = true; } } recordComment(TokenNameCOMMENT_LINE); if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition); if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) { if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) && this.lastPosition < this.currentPosition) { parseTags(); } if (this.recordLineSeparator) { if (isUnicode) { pushUnicodeLineSeparator(); } else { pushLineSeparator(); } } } if (this.tokenizeComments) { return TokenNameCOMMENT_LINE; } } catch (IndexOutOfBoundsException e) { this.currentPosition--; recordComment(TokenNameCOMMENT_LINE); if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition); if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) && this.lastPosition < this.currentPosition) { parseTags(); } if (this.tokenizeComments) { return TokenNameCOMMENT_LINE; } else { this.currentPosition++; } } break; } if (test > 0) { //traditional and javadoc comment try { //get the next char boolean isJavadoc = false, star = false; boolean isUnicode = false; int previous; // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; } else { isUnicode = false; if (this.withoutUnicodePtr != 0) { unicodeStore(); } } if (this.currentCharacter == '*') { isJavadoc = true; star = true; } if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) { if (this.recordLineSeparator) { if (isUnicode) { pushUnicodeLineSeparator(); } else { pushLineSeparator(); } } } isUnicode = false; previous = this.currentPosition; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { //-------------unicode traitement ------------ getNextUnicodeChar(); isUnicode = true; } else { isUnicode = false; } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; //jump over the \\ } // empty comment is not a javadoc /**/ if (this.currentCharacter == '/') { isJavadoc = false; } //loop until end of comment */ int firstTag = 0; while ((this.currentCharacter != '/') || (!star)) { if (this.currentPosition >= this.eofPosition) { throw new InvalidInputException(UNTERMINATED_COMMENT); } if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) { if (this.recordLineSeparator) { if (isUnicode) { pushUnicodeLineSeparator(); } else { pushLineSeparator(); } } } switch (this.currentCharacter) { case '*': star = true; break; case '@': if (firstTag == 0 && this.isFirstTag()) { firstTag = previous; } //$FALL-THROUGH$ default case to set star to false default: star = false; } //get next char previous = this.currentPosition; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { //-------------unicode traitement ------------ getNextUnicodeChar(); isUnicode = true; } else { isUnicode = false; } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; } //jump over the \\ } int token = isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK; recordComment(token); this.commentTagStarts[this.commentPtr] = firstTag; if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition); if (this.tokenizeComments) { /* if (isJavadoc) return TokenNameCOMMENT_JAVADOC; return TokenNameCOMMENT_BLOCK; */ return token; } } catch (IndexOutOfBoundsException e) { this.currentPosition--; throw new InvalidInputException(UNTERMINATED_COMMENT); } break; } } if (getNextChar('=')) return TokenNameDIVIDE_EQUAL; return TokenNameDIVIDE; case '\u001a' : if (atEnd()) return TokenNameEOF; //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$ default : char c = this.currentCharacter; if (c < ScannerHelper.MAX_OBVIOUS) { if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) { return scanIdentifierOrKeyword(); } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_DIGIT) != 0) { return scanNumber(false); } else { return TokenNameERROR; } } boolean isJavaIdStart; if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) { if (this.complianceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } // Unicode 4 detection char low = (char) getNextChar(); if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) { // illegal low surrogate throw new InvalidInputException(INVALID_LOW_SURROGATE); } isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c, low); } else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) { if (this.complianceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } throw new InvalidInputException(INVALID_HIGH_SURROGATE); } else { // optimized case already checked isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c); } if (isJavaIdStart) return scanIdentifierOrKeyword(); if (ScannerHelper.isDigit(this.currentCharacter)) { return scanNumber(false); } return TokenNameERROR; } } } //-----------------end switch while try-------------------- catch (IndexOutOfBoundsException e) { if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) { // reposition scanner in case we are interested by spaces as tokens this.currentPosition--; this.startPosition = whiteStart; return TokenNameWHITESPACE; } } return TokenNameEOF; } public void getNextUnicodeChar() throws InvalidInputException { //VOID //handle the case of unicode. //when a unicode appears then we must use a buffer that holds char internal values //At the end of this method currentCharacter holds the new visited char //and currentPosition points right next after it //ALL getNextChar.... ARE OPTIMIZED COPIES int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6; this.currentPosition++; if (this.currentPosition < this.eofPosition) { while (this.source[this.currentPosition] == 'u') { this.currentPosition++; if (this.currentPosition >= this.eofPosition) { this.currentPosition--; throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } unicodeSize++; } } else { this.currentPosition--; throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } if ((this.currentPosition + 4) > this.eofPosition) { this.currentPosition += (this.eofPosition - this.currentPosition); throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } if ((c1 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15 || c1 < 0 || (c2 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15 || c2 < 0 || (c3 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15 || c3 < 0 || (c4 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15 || c4 < 0){ throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4); //need the unicode buffer if (this.withoutUnicodePtr == 0) { //buffer all the entries that have been left aside.... unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition); } //fill the buffer with the char unicodeStore(); this.unicodeAsBackSlash = this.currentCharacter == '\\'; } public NLSTag[] getNLSTags() { final int length = this.nlsTagsPtr; if (length != 0) { NLSTag[] result = new NLSTag[length]; System.arraycopy(this.nlsTags, 0, result, 0, length); this.nlsTagsPtr = 0; return result; } return null; } public boolean[] getIdentityComparisonLines() { boolean [] retVal = this.validIdentityComparisonLines; this.validIdentityComparisonLines = null; return retVal; } public char[] getSource(){ return this.source; } protected boolean isFirstTag() { return true; } public final void jumpOverMethodBody() { this.wasAcr = false; int found = 1; try { while (true) { //loop for jumping over comments this.withoutUnicodePtr = 0; // ---------Consume white space and handles startPosition--------- boolean isWhiteSpace; do { this.startPosition = this.currentPosition; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { isWhiteSpace = jumpOverUnicodeWhiteSpace(); } else { if (this.recordLineSeparator && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n'))) { pushLineSeparator(); } isWhiteSpace = CharOperation.isWhitespace(this.currentCharacter); } } while (isWhiteSpace); // -------consume token until } is found--------- NextToken: switch (this.currentCharacter) { case '{' : found++; break NextToken; case '}' : found--; if (found == 0) return; break NextToken; case '\'' : { boolean test; test = getNextChar('\\'); if (test) { try { if (this.unicodeAsBackSlash) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } else { this.currentCharacter = this.source[this.currentPosition++]; } scanEscapeCharacter(); } catch (InvalidInputException ex) { // ignore } } else { try { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } catch (InvalidInputException ex) { // ignore } } getNextChar('\''); break NextToken; } case '"' : boolean isTextBlock = false; int firstClosingBrace = 0; try { try { // consume next character isTextBlock = scanForTextBlockBeginning(); if (!isTextBlock) { this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } } catch (InvalidInputException ex) { // ignore } Inner: while (this.currentPosition <= this.eofPosition) { if (isTextBlock) { switch (this.currentCharacter) { case '"': // look for text block delimiter if (scanForTextBlockClose()) { this.currentPosition += 2; this.currentCharacter = this.source[this.currentPosition]; isTextBlock = false; break Inner; } break; case '}': if (firstClosingBrace == 0) firstClosingBrace = this.currentPosition; break; case '\r' : if (this.source[this.currentPosition] == '\n') this.currentPosition++; //$FALL-THROUGH$ case '\n' : pushLineSeparator(); //$FALL-THROUGH$ default: if (this.currentCharacter == '\\' && this.source[this.currentPosition++] == '"') { this.currentPosition++; } this.currentCharacter = this.source[this.currentPosition++]; continue Inner; } } else if (this.currentCharacter == '"') { break Inner; } if (this.currentCharacter == '\r'){ if (this.source[this.currentPosition] == '\n') this.currentPosition++; break NextToken; // the string cannot go further that the line } if (this.currentCharacter == '\n'){ break; // the string cannot go further that the line } if (this.currentCharacter == '\\') { try { if (this.unicodeAsBackSlash) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } else { this.currentCharacter = this.source[this.currentPosition++]; } scanEscapeCharacter(); } catch (InvalidInputException ex) { // ignore } } try { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } catch (InvalidInputException ex) { // ignore } } } catch (IndexOutOfBoundsException e) { if(isTextBlock) { // Pull it back to the first closing brace after the beginning // of the unclosed text block and let recovery take over. if (firstClosingBrace > 0) { this.currentPosition = firstClosingBrace - 1; } } } break NextToken; case '/' : { int test; if ((test = getNextChar('/', '*')) == 0) { //line comment try { this.lastCommentLinePosition = this.currentPosition; //get the next char if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; } //jump over the \\ boolean isUnicode = false; while (this.currentCharacter != '\r' && this.currentCharacter != '\n') { if (this.currentPosition >= this.eofPosition) { this.lastCommentLinePosition = this.currentPosition; this.currentPosition ++; // this avoids duplicating the code inside the catch(IndexOutOfBoundsException e) below throw new IndexOutOfBoundsException(); } this.lastCommentLinePosition = this.currentPosition; //get the next char isUnicode = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { isUnicode = true; getNextUnicodeChar(); } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; } //jump over the \\ } /* * We need to completely consume the line break */ if (this.currentCharacter == '\r' && this.eofPosition > this.currentPosition) { if (this.source[this.currentPosition] == '\n') { this.currentPosition++; this.currentCharacter = '\n'; } else if ((this.source[this.currentPosition] == '\\') && (this.source[this.currentPosition + 1] == 'u')) { isUnicode = true; getNextUnicodeChar(); } } recordComment(TokenNameCOMMENT_LINE); if (this.recordLineSeparator && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n'))) { if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) && this.lastPosition < this.currentPosition) { parseTags(); } if (this.recordLineSeparator) { if (isUnicode) { pushUnicodeLineSeparator(); } else { pushLineSeparator(); } } } } catch (IndexOutOfBoundsException e) { //an eof will then be generated this.currentPosition--; recordComment(TokenNameCOMMENT_LINE); if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) && this.lastPosition < this.currentPosition) { parseTags(); } if (!this.tokenizeComments) { this.currentPosition++; } } break NextToken; } if (test > 0) { //traditional and javadoc comment boolean isJavadoc = false; try { //get the next char boolean star = false; int previous; boolean isUnicode = false; // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; } else { isUnicode = false; if (this.withoutUnicodePtr != 0) { unicodeStore(); } } if (this.currentCharacter == '*') { isJavadoc = true; star = true; } if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) { if (this.recordLineSeparator) { if (isUnicode) { pushUnicodeLineSeparator(); } else { pushLineSeparator(); } } } isUnicode = false; previous = this.currentPosition; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; } else { isUnicode = false; } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; //jump over the \\ } // empty comment is not a javadoc /**/ if (this.currentCharacter == '/') { isJavadoc = false; } //loop until end of comment */ int firstTag = 0; while ((this.currentCharacter != '/') || (!star)) { if (this.currentPosition >= this.eofPosition) { return; } if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) { if (this.recordLineSeparator) { if (isUnicode) { pushUnicodeLineSeparator(); } else { pushLineSeparator(); } } } switch (this.currentCharacter) { case '*': star = true; break; case '@': if (firstTag == 0 && this.isFirstTag()) { firstTag = previous; } //$FALL-THROUGH$ default case to set star to false default: star = false; } //get next char previous = this.currentPosition; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); isUnicode = true; } else { isUnicode = false; } //handle the \\u case manually into comment if (this.currentCharacter == '\\') { if (this.source[this.currentPosition] == '\\') this.currentPosition++; } //jump over the \\ } recordComment(isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK); this.commentTagStarts[this.commentPtr] = firstTag; } catch (IndexOutOfBoundsException e) { return; } break NextToken; } break NextToken; } default : try { char c = this.currentCharacter; if (c < ScannerHelper.MAX_OBVIOUS) { if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) { scanIdentifierOrKeyword(); break NextToken; } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_DIGIT) != 0) { scanNumber(false); break NextToken; } else { break NextToken; } } boolean isJavaIdStart; if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) { if (this.complianceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(INVALID_UNICODE_ESCAPE); } // Unicode 4 detection char low = (char) getNextChar(); if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) { // illegal low surrogate break NextToken; } isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c, low); } else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) { break NextToken; } else { // optimized case already checked isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c); } if (isJavaIdStart) { scanIdentifierOrKeyword(); break NextToken; } // if (ScannerHelper.isDigit(this.currentCharacter)) { // scanNumber(false); // break NextToken; // } } catch (InvalidInputException ex) { // ignore } } } //-----------------end switch while try-------------------- } catch (IndexOutOfBoundsException | InvalidInputException e) { // ignore } return; } public final boolean jumpOverUnicodeWhiteSpace() throws InvalidInputException { //BOOLEAN //handle the case of unicode. Jump over the next whiteSpace //making startPosition pointing on the next available char //On false, the currentCharacter is filled up with a potential //correct char this.wasAcr = false; getNextUnicodeChar(); return CharOperation.isWhitespace(this.currentCharacter); } final char[] optimizedCurrentTokenSource1() { //return always the same char[] build only once //optimization at no speed cost of 99.5 % of the singleCharIdentifier char charOne = this.source[this.startPosition]; switch (charOne) { case 'a' : return charArray_a; case 'b' : return charArray_b; case 'c' : return charArray_c; case 'd' : return charArray_d; case 'e' : return charArray_e; case 'f' : return charArray_f; case 'g' : return charArray_g; case 'h' : return charArray_h; case 'i' : return charArray_i; case 'j' : return charArray_j; case 'k' : return charArray_k; case 'l' : return charArray_l; case 'm' : return charArray_m; case 'n' : return charArray_n; case 'o' : return charArray_o; case 'p' : return charArray_p; case 'q' : return charArray_q; case 'r' : return charArray_r; case 's' : return charArray_s; case 't' : return charArray_t; case 'u' : return charArray_u; case 'v' : return charArray_v; case 'w' : return charArray_w; case 'x' : return charArray_x; case 'y' : return charArray_y; case 'z' : return charArray_z; default : return new char[] {charOne}; } } final char[] optimizedCurrentTokenSource2() { //try to return the same char[] build only once char[] src = this.source; int start = this.startPosition; char c0 , c1; int hash = (((c0=src[start]) << 6) + (c1=src[start+1])) % TableSize; char[][] table = this.charArray_length[0][hash]; int i = this.newEntry2; while (++i < InternalTableSize) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1])) return charArray; } //---------other side--------- i = -1; int max = this.newEntry2; while (++i <= max) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1])) return charArray; } //--------add the entry------- if (++max >= InternalTableSize) max = 0; char[] r; System.arraycopy(src, start, r= new char[2], 0, 2); //newIdentCount++; return table[this.newEntry2 = max] = r; //(r = new char[] {c0, c1}); } final char[] optimizedCurrentTokenSource3() { //try to return the same char[] build only once char[] src = this.source; int start = this.startPosition; char c0, c1=src[start+1], c2; int hash = (((c0=src[start])<< 6) + (c2=src[start+2])) % TableSize; // int hash = ((c0 << 12) + (c1<< 6) + c2) % TableSize; char[][] table = this.charArray_length[1][hash]; int i = this.newEntry3; while (++i < InternalTableSize) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) return charArray; } //---------other side--------- i = -1; int max = this.newEntry3; while (++i <= max) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) return charArray; } //--------add the entry------- if (++max >= InternalTableSize) max = 0; char[] r; System.arraycopy(src, start, r= new char[3], 0, 3); //newIdentCount++; return table[this.newEntry3 = max] = r; //(r = new char[] {c0, c1, c2}); } final char[] optimizedCurrentTokenSource4() { //try to return the same char[] build only once char[] src = this.source; int start = this.startPosition; char c0, c1 = src[start+1], c2, c3 = src[start+3]; int hash = (((c0=src[start]) << 6) + (c2=src[start+2])) % TableSize; // int hash = (int) (((((long) c0) << 18) + (c1 << 12) + (c2 << 6) + c3) % TableSize); char[][] table = this.charArray_length[2][hash]; int i = this.newEntry4; while (++i < InternalTableSize) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3])) return charArray; } //---------other side--------- i = -1; int max = this.newEntry4; while (++i <= max) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3])) return charArray; } //--------add the entry------- if (++max >= InternalTableSize) max = 0; char[] r; System.arraycopy(src, start, r= new char[4], 0, 4); //newIdentCount++; return table[this.newEntry4 = max] = r; //(r = new char[] {c0, c1, c2, c3}); } final char[] optimizedCurrentTokenSource5() { //try to return the same char[] build only once char[] src = this.source; int start = this.startPosition; char c0, c1 = src[start+1], c2, c3 = src[start+3], c4; int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize; // int hash = (int) (((((long) c0) << 24) + (((long) c1) << 18) + (c2 << 12) + (c3 << 6) + c4) % TableSize); char[][] table = this.charArray_length[3][hash]; int i = this.newEntry5; while (++i < InternalTableSize) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) && (c4 == charArray[4])) return charArray; } //---------other side--------- i = -1; int max = this.newEntry5; while (++i <= max) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) && (c4 == charArray[4])) return charArray; } //--------add the entry------- if (++max >= InternalTableSize) max = 0; char[] r; System.arraycopy(src, start, r= new char[5], 0, 5); //newIdentCount++; return table[this.newEntry5 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4}); } final char[] optimizedCurrentTokenSource6() { //try to return the same char[] build only once char[] src = this.source; int start = this.startPosition; char c0, c1 = src[start+1], c2, c3 = src[start+3], c4, c5 = src[start+5]; int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize; // int hash = (int)(((((long) c0) << 32) + (((long) c1) << 24) + (((long) c2) << 18) + (c3 << 12) + (c4 << 6) + c5) % TableSize); char[][] table = this.charArray_length[4][hash]; int i = this.newEntry6; while (++i < InternalTableSize) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) && (c4 == charArray[4]) && (c5 == charArray[5])) return charArray; } //---------other side--------- i = -1; int max = this.newEntry6; while (++i <= max) { char[] charArray = table[i]; if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) && (c4 == charArray[4]) && (c5 == charArray[5])) return charArray; } //--------add the entry------- if (++max >= InternalTableSize) max = 0; char[] r; System.arraycopy(src, start, r= new char[6], 0, 6); //newIdentCount++; return table[this.newEntry6 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4, c5}); } public boolean isInModuleDeclaration() { return this.fakeInModule || this.insideModuleInfo || (this.activeParser != null ? this.activeParser.isParsingModuleDeclaration() : false); } protected boolean areRestrictedModuleKeywordsActive() { return this.scanContext != null && this.scanContext != ScanContext.INACTIVE; } void updateScanContext(int token) { switch (token) { case TerminalTokens.TokenNameSEMICOLON: // next could be a KEYWORD case TerminalTokens.TokenNameRBRACE: case TokenNameRPAREN: this.scanContext = ScanContext.EXPECTING_KEYWORD; break; case TokenNameopen: this.scanContext = ScanContext.EXPECTING_KEYWORD; break; case TokenNamerequires: this.scanContext = ScanContext.AFTER_REQUIRES; break; case TokenNamemodule: case TokenNameexports: case TokenNameopens: case TokenNameuses: case TokenNameprovides: case TokenNameto: case TokenNamewith: case TokenNametransitive: case TokenNameDOT: case TokenNameimport: case TokenNameAT: case TokenNameAT308: case TokenNameCOMMA: this.scanContext = ScanContext.EXPECTING_IDENTIFIER; break; case TokenNameIdentifier: this.scanContext = ScanContext.EXPECTING_KEYWORD; break; case TerminalTokens.TokenNameLBRACE: this.scanContext = ScanContext.EXPECTING_KEYWORD; break; default: // anything else is unexpected and should not alter the context break; } } private void parseTags() { int position = 0; final int currentStartPosition = this.startPosition; final int currentLinePtr = this.linePtr; if (currentLinePtr >= 0) { position = this.lineEnds[currentLinePtr] + 1; } while (ScannerHelper.isWhitespace(this.source[position])) { position++; } if (currentStartPosition == position) { // the whole line is commented out return; } char[] s = null; int sourceEnd = this.currentPosition; int sourceStart = currentStartPosition; int sourceDelta = 0; if (this.withoutUnicodePtr != 0) { // 0 is used as a fast test flag so the real first char is in position 1 System.arraycopy( this.withoutUnicodeBuffer, 1, s = new char[this.withoutUnicodePtr], 0, this.withoutUnicodePtr); sourceEnd = this.withoutUnicodePtr; sourceStart = 1; sourceDelta = currentStartPosition; } else { s = this.source; } int pos; if (this.checkNonExternalizedStringLiterals && (pos = CharOperation.indexOf(TAG_PREFIX, s, true, sourceStart, sourceEnd)) != -1) { if (this.nlsTags == null) { this.nlsTags = new NLSTag[10]; this.nlsTagsPtr = 0; } while (pos != -1) { int start = pos + TAG_PREFIX_LENGTH; int end = CharOperation.indexOf(TAG_POSTFIX, s, start, sourceEnd); if (end != -1) { NLSTag currentTag = null; final int currentLine = currentLinePtr + 1; try { currentTag = new NLSTag(pos + sourceDelta, end + sourceDelta, currentLine, extractInt(s, start, end)); } catch (NumberFormatException e) { currentTag = new NLSTag(pos + sourceDelta, end + sourceDelta, currentLine, -1); } if (this.nlsTagsPtr == this.nlsTags.length) { // resize System.arraycopy(this.nlsTags, 0, (this.nlsTags = new NLSTag[this.nlsTagsPtr + 10]), 0, this.nlsTagsPtr); } this.nlsTags[this.nlsTagsPtr++] = currentTag; } else { end = start; } pos = CharOperation.indexOf(TAG_PREFIX, s, true, end, sourceEnd); } } if (this.checkUninternedIdentityComparison && (pos = CharOperation.indexOf(IDENTITY_COMPARISON_TAG, s, true, sourceStart, sourceEnd)) != -1) { if (this.validIdentityComparisonLines == null) { this.validIdentityComparisonLines = new boolean[0]; } int currentLine = currentLinePtr + 1; int length = this.validIdentityComparisonLines.length; System.arraycopy(this.validIdentityComparisonLines, 0, this.validIdentityComparisonLines = new boolean[currentLine + 1], 0, length); this.validIdentityComparisonLines[currentLine] = true; } } private int extractInt(char[] array, int start, int end) { int value = 0; for (int i = start; i < end; i++) { final char currentChar = array[i]; int digit = 0; switch(currentChar) { case '0' : digit = 0; break; case '1' : digit = 1; break; case '2' : digit = 2; break; case '3' : digit = 3; break; case '4' : digit = 4; break; case '5' : digit = 5; break; case '6' : digit = 6; break; case '7' : digit = 7; break; case '8' : digit = 8; break; case '9' : digit = 9; break; default : throw new NumberFormatException(); } value *= 10; if (digit < 0) throw new NumberFormatException(); value += digit; } return value; } public final void pushLineSeparator() { //see comment on isLineDelimiter(char) for the use of '\n' and '\r' final int INCREMENT = 250; //currentCharacter is at position currentPosition-1 // cr 000D if (this.currentCharacter == '\r') { int separatorPos = this.currentPosition - 1; if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return; int length = this.lineEnds.length; if (++this.linePtr >= length) System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[2*length + INCREMENT], 0, length); this.lineEnds[this.linePtr] = separatorPos; // look-ahead for merged cr+lf try { if (this.source[this.currentPosition] == '\n') { //System.out.println("look-ahead LF-" + this.currentPosition); this.lineEnds[this.linePtr] = this.currentPosition; this.currentPosition++; this.wasAcr = false; } else { this.wasAcr = true; } } catch(IndexOutOfBoundsException e) { this.wasAcr = true; } } else { // lf 000A if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf if (this.wasAcr && (this.lineEnds[this.linePtr] == (this.currentPosition - 2))) { //System.out.println("merge LF-" + (this.currentPosition - 1)); this.lineEnds[this.linePtr] = this.currentPosition - 1; } else { int separatorPos = this.currentPosition - 1; if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return; int length = this.lineEnds.length; if (++this.linePtr >= length) System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[2*length + INCREMENT], 0, length); this.lineEnds[this.linePtr] = separatorPos; } this.wasAcr = false; } } } public final void pushUnicodeLineSeparator() { // cr 000D if (this.currentCharacter == '\r') { if (this.source[this.currentPosition] == '\n') { this.wasAcr = false; } else { this.wasAcr = true; } } else { // lf 000A if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf this.wasAcr = false; } } } public void recordComment(int token) { // compute position int commentStart = this.startPosition; int stopPosition = this.currentPosition; switch (token) { case TokenNameCOMMENT_LINE: // both positions are negative commentStart = -this.startPosition; stopPosition = -this.lastCommentLinePosition; break; case TokenNameCOMMENT_BLOCK: // only end position is negative stopPosition = -this.currentPosition; break; } // a new comment is recorded int length = this.commentStops.length; if (++this.commentPtr >= length) { int newLength = length + COMMENT_ARRAYS_SIZE*10; System.arraycopy(this.commentStops, 0, this.commentStops = new int[newLength], 0, length); System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[newLength], 0, length); System.arraycopy(this.commentTagStarts, 0, this.commentTagStarts = new int[newLength], 0, length); } this.commentStops[this.commentPtr] = stopPosition; this.commentStarts[this.commentPtr] = commentStart; }
Reposition the scanner on some portion of the original source. The given endPosition is the last valid position. Beyond this position, the scanner will answer EOF tokens (ITerminalSymbols.TokenNameEOF).
Params:
  • begin – the given start position
  • end – the given end position
/** * Reposition the scanner on some portion of the original source. The given endPosition is the last valid position. * Beyond this position, the scanner will answer EOF tokens (<code>ITerminalSymbols.TokenNameEOF</code>). * * @param begin the given start position * @param end the given end position */
public void resetTo(int begin, int end) { resetTo(begin, end, isInModuleDeclaration()); } public void resetTo(int begin, int end, boolean isModuleInfo) { resetTo(begin, end, isModuleInfo, null); }
Reposition the scanner on some portion of the original source. The given endPosition is the last valid position. Beyond this position, the scanner will answer EOF tokens (ITerminalSymbols.TokenNameEOF).
Params:
  • begin – the given start position
  • end – the given end position
  • isModuleInfo – if true apply rules for restricted keywords even without a connection to a properly configured parser
  • context – The scan context to use for restricted keyword support, use null to compute
/** * Reposition the scanner on some portion of the original source. The given endPosition is the last valid position. * Beyond this position, the scanner will answer EOF tokens (<code>ITerminalSymbols.TokenNameEOF</code>). * * @param begin the given start position * @param end the given end position * @param isModuleInfo if true apply rules for restricted keywords even without a connection to a properly configured parser * @param context The scan context to use for restricted keyword support, use null to compute */
public void resetTo(int begin, int end, boolean isModuleInfo, ScanContext context) { //reset the scanner to a given position where it may rescan again this.diet = false; this.initialPosition = this.startPosition = this.currentPosition = begin; if (this.source != null && this.source.length < end) { this.eofPosition = this.source.length; } else { this.eofPosition = end < Integer.MAX_VALUE ? end + 1 : end; } this.commentPtr = -1; // reset comment stack this.foundTaskCount = 0; this.lookBack[0] = this.lookBack[1] = this.nextToken = TokenNameNotAToken; this.consumingEllipsisAnnotations = false; this.insideModuleInfo = isModuleInfo; this.scanContext = context == null ? getScanContext(begin) : context; } private ScanContext getScanContext(int begin) { if (!isInModuleDeclaration()) return ScanContext.INACTIVE; if (begin == 0) return ScanContext.EXPECTING_KEYWORD; CompilerOptions options = new CompilerOptions(); options.complianceLevel = this.complianceLevel; options.sourceLevel = this.sourceLevel; ScanContextDetector parser = new ScanContextDetector(options); return parser.getScanContext(this.source, begin - 1); } protected final void scanEscapeCharacter() throws InvalidInputException { // the string with "\\u" is a legal string of two chars \ and u //thus we use a direct access to the source (for regular cases). switch (this.currentCharacter) { case 'b' : this.currentCharacter = '\b'; break; case 't' : this.currentCharacter = '\t'; break; case 'n' : this.currentCharacter = '\n'; break; case 'f' : this.currentCharacter = '\f'; break; case 'r' : this.currentCharacter = '\r'; break; case '\"' : this.currentCharacter = '\"'; break; case '\'' : this.currentCharacter = '\''; break; case '\\' : this.currentCharacter = '\\'; break; default : // -----------octal escape-------------- // OctalDigit // OctalDigit OctalDigit // ZeroToThree OctalDigit OctalDigit int number = ScannerHelper.getHexadecimalValue(this.currentCharacter); if (number >= 0 && number <= 7) { boolean zeroToThreeNot = number > 3; if (ScannerHelper.isDigit(this.currentCharacter = this.source[this.currentPosition++])) { int digit = ScannerHelper.getHexadecimalValue(this.currentCharacter); if (digit >= 0 && digit <= 7) { number = (number * 8) + digit; if (ScannerHelper.isDigit(this.currentCharacter = this.source[this.currentPosition++])) { if (zeroToThreeNot) {// has read \NotZeroToThree OctalDigit Digit --> ignore last character this.currentPosition--; } else { digit = ScannerHelper.getHexadecimalValue(this.currentCharacter); if (digit >= 0 && digit <= 7){ // has read \ZeroToThree OctalDigit OctalDigit number = (number * 8) + digit; } else {// has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character this.currentPosition--; } } } else { // has read \OctalDigit NonDigit--> ignore last character this.currentPosition--; } } else { // has read \OctalDigit NonOctalDigit--> ignore last character this.currentPosition--; } } else { // has read \OctalDigit --> ignore last character this.currentPosition--; } if (number > 255) throw new InvalidInputException(INVALID_ESCAPE); this.currentCharacter = (char) number; } else throw new InvalidInputException(INVALID_ESCAPE); } } public int scanIdentifierOrKeywordWithBoundCheck() { //test keywords //first dispatch on the first char. //then the length. If there are several //keywors with the same length AND the same first char, then do another //dispatch on the second char this.useAssertAsAnIndentifier = false; this.useEnumAsAnIndentifier = false; char[] src = this.source; identLoop: { int pos; int srcLength = this.eofPosition; while (true) { if ((pos = this.currentPosition) >= srcLength) // handle the obvious case upfront break identLoop; char c = src[pos]; if (c < ScannerHelper.MAX_OBVIOUS) { if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_UPPER_LETTER | ScannerHelper.C_LOWER_LETTER | ScannerHelper.C_IDENT_PART | ScannerHelper.C_DIGIT)) != 0) { if (this.withoutUnicodePtr != 0) { this.currentCharacter = c; unicodeStore(); } this.currentPosition++; } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_SEPARATOR | ScannerHelper.C_JLS_SPACE)) != 0) { this.currentCharacter = c; break identLoop; } else { //System.out.println("slow<=128: "+ c); while (getNextCharAsJavaIdentifierPartWithBoundCheck()){/*empty*/} break identLoop; } } else { //System.out.println("slow>>128: "+ c); while (getNextCharAsJavaIdentifierPartWithBoundCheck()){/*empty*/} break identLoop; } } } int index, length; char[] data; if (this.withoutUnicodePtr == 0) { //quick test on length == 1 but not on length > 12 while most identifier //have a length which is <= 12...but there are lots of identifier with //only one char.... if ((length = this.currentPosition - this.startPosition) == 1) { return TokenNameIdentifier; } data = this.source; index = this.startPosition; } else { if ((length = this.withoutUnicodePtr) == 1) return TokenNameIdentifier; data = this.withoutUnicodeBuffer; index = 1; } return internalScanIdentifierOrKeyword(index, length, data); } public int scanIdentifierOrKeyword() { //test keywords //first dispatch on the first char. //then the length. If there are several //keywords with the same length AND the same first char, then do another //dispatch on the second char this.useAssertAsAnIndentifier = false; this.useEnumAsAnIndentifier = false; char[] src = this.source; identLoop: { int pos; int srcLength = this.eofPosition; while (true) { if ((pos = this.currentPosition) >= srcLength) // handle the obvious case upfront break identLoop; char c = src[pos]; if (c < ScannerHelper.MAX_OBVIOUS) { if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_UPPER_LETTER | ScannerHelper.C_LOWER_LETTER | ScannerHelper.C_IDENT_PART | ScannerHelper.C_DIGIT)) != 0) { if (this.withoutUnicodePtr != 0) { this.currentCharacter = c; unicodeStore(); } this.currentPosition++; } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_SEPARATOR | ScannerHelper.C_JLS_SPACE)) != 0) { this.currentCharacter = c; break identLoop; } else { //System.out.println("slow<=128: "+ c); while (getNextCharAsJavaIdentifierPart()){/*empty*/} break identLoop; } } else { //System.out.println("slow>>128: "+ c); while (getNextCharAsJavaIdentifierPart()){/*empty*/} break identLoop; } } } int index, length; char[] data; if (this.withoutUnicodePtr == 0) { //quick test on length == 1 but not on length > 12 while most identifier //have a length which is <= 12...but there are lots of identifier with //only one char.... if ((length = this.currentPosition - this.startPosition) == 1) { return TokenNameIdentifier; } data = this.source; index = this.startPosition; } else { if ((length = this.withoutUnicodePtr) == 1) return TokenNameIdentifier; data = this.withoutUnicodeBuffer; index = 1; } return internalScanIdentifierOrKeyword(index, length, data); } private int internalScanIdentifierOrKeyword(int index, int length, char[] data) { switch (data[index]) { case 'a' : switch(length) { case 8: //abstract if ((data[++index] == 'b') && (data[++index] == 's') && (data[++index] == 't') && (data[++index] == 'r') && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 't')) { return TokenNameabstract; } else { return TokenNameIdentifier; } case 6: // assert if ((data[++index] == 's') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 'r') && (data[++index] == 't')) { if (this.sourceLevel >= ClassFileConstants.JDK1_4) { this.containsAssertKeyword = true; return TokenNameassert; } else { this.useAssertAsAnIndentifier = true; return TokenNameIdentifier; } } else { return TokenNameIdentifier; } default: return TokenNameIdentifier; } case 'b' : //boolean break byte switch (length) { case 4 : if ((data[++index] == 'y') && (data[++index] == 't') && (data[++index] == 'e')) return TokenNamebyte; else return TokenNameIdentifier; case 5 : if ((data[++index] == 'r') && (data[++index] == 'e') && (data[++index] == 'a') && (data[++index] == 'k')) return TokenNamebreak; else return TokenNameIdentifier; case 7 : if ((data[++index] == 'o') && (data[++index] == 'o') && (data[++index] == 'l') && (data[++index] == 'e') && (data[++index] == 'a') && (data[++index] == 'n')) return TokenNameboolean; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'c' : //case char catch const class continue switch (length) { case 4 : if (data[++index] == 'a') if ((data[++index] == 's') && (data[++index] == 'e')) return TokenNamecase; else return TokenNameIdentifier; else if ((data[index] == 'h') && (data[++index] == 'a') && (data[++index] == 'r')) return TokenNamechar; else return TokenNameIdentifier; case 5 : if (data[++index] == 'a') if ((data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h')) return TokenNamecatch; else return TokenNameIdentifier; else if (data[index] == 'l') if ((data[++index] == 'a') && (data[++index] == 's') && (data[++index] == 's')) return TokenNameclass; else return TokenNameIdentifier; else if ((data[index] == 'o') && (data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 't')) return TokenNameconst; //const is not used in java ??????? else return TokenNameIdentifier; case 8 : if ((data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 'u') && (data[++index] == 'e')) return TokenNamecontinue; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'd' : //default do double switch (length) { case 2 : if ((data[++index] == 'o')) return TokenNamedo; else return TokenNameIdentifier; case 6 : if ((data[++index] == 'o') && (data[++index] == 'u') && (data[++index] == 'b') && (data[++index] == 'l') && (data[++index] == 'e')) return TokenNamedouble; else return TokenNameIdentifier; case 7 : if ((data[++index] == 'e') && (data[++index] == 'f') && (data[++index] == 'a') && (data[++index] == 'u') && (data[++index] == 'l') && (data[++index] == 't')) return TokenNamedefault; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'e' : //else extends exports switch (length) { case 4 : if (data[++index] == 'l') { if ((data[++index] == 's') && (data[++index] == 'e')) { return TokenNameelse; } else { return TokenNameIdentifier; } } else if ((data[index] == 'n') && (data[++index] == 'u') && (data[++index] == 'm')) { if (this.sourceLevel >= ClassFileConstants.JDK1_5) { return TokenNameenum; } else { this.useEnumAsAnIndentifier = true; return TokenNameIdentifier; } } return TokenNameIdentifier; case 7 : if ((data[++index] == 'x')) { if ((data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 's')) { return TokenNameextends; } else if (areRestrictedModuleKeywordsActive() && (data[index] == 'p') && (data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 't') && (data[++index] == 's')) { return TokenNameexports; } else return TokenNameIdentifier; } else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'f' : //final finally float for false switch (length) { case 3 : if ((data[++index] == 'o') && (data[++index] == 'r')) return TokenNamefor; else return TokenNameIdentifier; case 5 : if (data[++index] == 'i') if ((data[++index] == 'n') && (data[++index] == 'a') && (data[++index] == 'l')) { return TokenNamefinal; } else return TokenNameIdentifier; else if (data[index] == 'l') if ((data[++index] == 'o') && (data[++index] == 'a') && (data[++index] == 't')) return TokenNamefloat; else return TokenNameIdentifier; else if ((data[index] == 'a') && (data[++index] == 'l') && (data[++index] == 's') && (data[++index] == 'e')) return TokenNamefalse; else return TokenNameIdentifier; case 7 : if ((data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 'a') && (data[++index] == 'l') && (data[++index] == 'l') && (data[++index] == 'y')) return TokenNamefinally; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'g' : //goto if (length == 4) { if ((data[++index] == 'o') && (data[++index] == 't') && (data[++index] == 'o')) { return TokenNamegoto; } } //no goto in java are allowed, so why java removes this keyword ??? return TokenNameIdentifier; case 'i' : //if implements import instanceof int interface switch (length) { case 2 : if (data[++index] == 'f') return TokenNameif; else return TokenNameIdentifier; case 3 : if ((data[++index] == 'n') && (data[++index] == 't')) return TokenNameint; else return TokenNameIdentifier; case 6 : if ((data[++index] == 'm') && (data[++index] == 'p') && (data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 't')) return TokenNameimport; else return TokenNameIdentifier; case 9 : if ((data[++index] == 'n') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'r') && (data[++index] == 'f') && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 'e')) return TokenNameinterface; else return TokenNameIdentifier; case 10 : if (data[++index] == 'm') if ((data[++index] == 'p') && (data[++index] == 'l') && (data[++index] == 'e') && (data[++index] == 'm') && (data[++index] == 'e') && (data[++index] == 'n') && (data[++index] == 't') && (data[++index] == 's')) return TokenNameimplements; else return TokenNameIdentifier; else if ((data[index] == 'n') && (data[++index] == 's') && (data[++index] == 't') && (data[++index] == 'a') && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'e') && (data[++index] == 'o') && (data[++index] == 'f')) return TokenNameinstanceof; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'l' : //long if (length == 4) { if ((data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 'g')) { return TokenNamelong; } } return TokenNameIdentifier; case 'm': //module switch (length) { case 6 : if (areRestrictedModuleKeywordsActive() && (data[++index] == 'o') && (data[++index] == 'd') && (data[++index] == 'u') && (data[++index] == 'l') && (data[++index] == 'e')) return TokenNamemodule; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'n' : //native new null switch (length) { case 3 : if ((data[++index] == 'e') && (data[++index] == 'w')) return TokenNamenew; else return TokenNameIdentifier; case 4 : if ((data[++index] == 'u') && (data[++index] == 'l') && (data[++index] == 'l')) return TokenNamenull; else return TokenNameIdentifier; case 6 : if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'v') && (data[++index] == 'e')) { return TokenNamenative; } else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'o': switch (length) { case 4 : if (areRestrictedModuleKeywordsActive() && (data[++index] == 'p') && (data[++index] == 'e') && (data[++index] == 'n')) return TokenNameopen; else return TokenNameIdentifier; case 5 : if (areRestrictedModuleKeywordsActive() && (data[++index] == 'p') && (data[++index] == 'e') && (data[++index] == 'n') && (data[++index] == 's')) return TokenNameopens; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'p' : //package private protected public provides switch (length) { case 6 : if ((data[++index] == 'u') && (data[++index] == 'b') && (data[++index] == 'l') && (data[++index] == 'i') && (data[++index] == 'c')) { return TokenNamepublic; } else return TokenNameIdentifier; case 7 : if (data[++index] == 'a') if ((data[++index] == 'c') && (data[++index] == 'k') && (data[++index] == 'a') && (data[++index] == 'g') && (data[++index] == 'e')) return TokenNamepackage; else return TokenNameIdentifier; else if ((data[index] == 'r') && (data[++index] == 'i') && (data[++index] == 'v') && (data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'e')) { return TokenNameprivate; } else return TokenNameIdentifier; case 8 : if (areRestrictedModuleKeywordsActive() && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'v') && (data[++index] == 'i') && (data[++index] == 'd') && (data[++index] == 'e') && (data[++index] == 's')) { return TokenNameprovides; } else return TokenNameIdentifier; case 9 : if ((data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'c') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'd')) { return TokenNameprotected; } else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'r' : //return requires switch (length) { case 6: if ((data[++index] == 'e') && (data[++index] == 't') && (data[++index] == 'u') && (data[++index] == 'r') && (data[++index] == 'n')) { return TokenNamereturn; } else return TokenNameIdentifier; case 8: if (areRestrictedModuleKeywordsActive() && (data[++index] == 'e') && (data[++index] == 'q') && (data[++index] == 'u') && (data[++index] == 'i') && (data[++index] == 'r') && (data[++index] == 'e') && (data[++index] == 's')) { return TokenNamerequires; } else return TokenNameIdentifier; } return TokenNameIdentifier; case 's' : //short static super switch synchronized strictfp switch (length) { case 5 : if (data[++index] == 'h') if ((data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 't')) return TokenNameshort; else return TokenNameIdentifier; else if ((data[index] == 'u') && (data[++index] == 'p') && (data[++index] == 'e') && (data[++index] == 'r')) return TokenNamesuper; else return TokenNameIdentifier; case 6 : if (data[++index] == 't') if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'c')) { return TokenNamestatic; } else return TokenNameIdentifier; else if ((data[index] == 'w') && (data[++index] == 'i') && (data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h')) return TokenNameswitch; else return TokenNameIdentifier; case 8 : if ((data[++index] == 't') && (data[++index] == 'r') && (data[++index] == 'i') && (data[++index] == 'c') && (data[++index] == 't') && (data[++index] == 'f') && (data[++index] == 'p')) return TokenNamestrictfp; else return TokenNameIdentifier; case 12 : if ((data[++index] == 'y') && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'h') && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 'i') && (data[++index] == 'z') && (data[++index] == 'e') && (data[++index] == 'd')) { return TokenNamesynchronized; } else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 't' : //try throw throws transient this true switch (length) { case 2: if (areRestrictedModuleKeywordsActive() && data[++index] == 'o') return TokenNameto; else return TokenNameIdentifier; case 3 : if ((data[++index] == 'r') && (data[++index] == 'y')) return TokenNametry; else return TokenNameIdentifier; case 4 : if (data[++index] == 'h') if ((data[++index] == 'i') && (data[++index] == 's')) return TokenNamethis; else return TokenNameIdentifier; else if ((data[index] == 'r') && (data[++index] == 'u') && (data[++index] == 'e')) return TokenNametrue; else return TokenNameIdentifier; case 5 : if ((data[++index] == 'h') && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'w')) return TokenNamethrow; else return TokenNameIdentifier; case 6 : if ((data[++index] == 'h') && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'w') && (data[++index] == 's')) return TokenNamethrows; else return TokenNameIdentifier; case 9 : if ((data[++index] == 'r') && (data[++index] == 'a') && (data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 'i') && (data[++index] == 'e') && (data[++index] == 'n') && (data[++index] == 't')) { return TokenNametransient; } else return TokenNameIdentifier; case 10: if (areRestrictedModuleKeywordsActive() && (data[++index] == 'r') && (data[++index] == 'a') && (data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 'i') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'v') && (data[++index] == 'e')) { return TokenNametransitive; } else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'u' : //uses switch(length) { case 4 : if (areRestrictedModuleKeywordsActive() && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 's')) return TokenNameuses; else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'v' : //void volatile switch (length) { case 4 : if ((data[++index] == 'o') && (data[++index] == 'i') && (data[++index] == 'd')) return TokenNamevoid; else return TokenNameIdentifier; case 8 : if ((data[++index] == 'o') && (data[++index] == 'l') && (data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'l') && (data[++index] == 'e')) { return TokenNamevolatile; } else return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'w' : //while widefp with switch (length) { case 4: if (areRestrictedModuleKeywordsActive() && (data[++index] == 'i') && (data[++index] == 't') && (data[++index] == 'h')) return TokenNamewith; else return TokenNameIdentifier; case 5 : if ((data[++index] == 'h') && (data[++index] == 'i') && (data[++index] == 'l') && (data[++index] == 'e')) return TokenNamewhile; else return TokenNameIdentifier; //case 6:if ( (data[++index] =='i') && (data[++index]=='d') && (data[++index]=='e') && (data[++index]=='f')&& (data[++index]=='p')) //return TokenNamewidefp ; //else //return TokenNameIdentifier; default : return TokenNameIdentifier; } case 'y' : switch (length) { case 5 : if ((data[++index] == 'i') && (data[++index] == 'e') && (data[++index] == 'l') && (data[++index] == 'd')) return disambiguatedRestrictedIdentifierYield(TokenNameRestrictedIdentifierYield); //$FALL-THROUGH$ default : return TokenNameIdentifier; } default : return TokenNameIdentifier; } } public int scanNumber(boolean dotPrefix) throws InvalidInputException { //when entering this method the currentCharacter is the first //digit of the number. It may be preceeded by a '.' when //dotPrefix is true boolean floating = dotPrefix; if (!dotPrefix && (this.currentCharacter == '0')) { if (getNextChar('x', 'X') >= 0) { //----------hexa----------------- int start = this.currentPosition; consumeDigits(16, true); int end = this.currentPosition; if (getNextChar('l', 'L') >= 0) { if (end == start) { throw new InvalidInputException(INVALID_HEXA); } return TokenNameLongLiteral; } else if (getNextChar('.')) { // hexadecimal floating point literal // read decimal part boolean hasNoDigitsBeforeDot = end == start; start = this.currentPosition; consumeDigits(16, true); end = this.currentPosition; if (hasNoDigitsBeforeDot && end == start) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } throw new InvalidInputException(INVALID_HEXA); } if (getNextChar('p', 'P') >= 0) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } if ((this.currentCharacter == '-') || (this.currentCharacter == '+')) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } if (!ScannerHelper.isDigit(this.currentCharacter)) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } if (this.currentCharacter == '_') { // wrongly place '_' consumeDigits(10); throw new InvalidInputException(INVALID_UNDERSCORE); } throw new InvalidInputException(INVALID_HEXA); } consumeDigits(10); if (getNextChar('f', 'F') >= 0) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } return TokenNameFloatingPointLiteral; } if (getNextChar('d', 'D') >= 0) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } return TokenNameDoubleLiteral; } if (getNextChar('l', 'L') >= 0) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } throw new InvalidInputException(INVALID_HEXA); } if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } return TokenNameDoubleLiteral; } else { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } throw new InvalidInputException(INVALID_HEXA); } } else if (getNextChar('p', 'P') >= 0) { // consume next character if (end == start) { // Has no digits before exponent if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } throw new InvalidInputException(INVALID_HEXA); } this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } if ((this.currentCharacter == '-') || (this.currentCharacter == '+')) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } if (!ScannerHelper.isDigit(this.currentCharacter)) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } if (this.currentCharacter == '_') { // wrongly place '_' consumeDigits(10); throw new InvalidInputException(INVALID_UNDERSCORE); } throw new InvalidInputException(INVALID_FLOAT); } consumeDigits(10); if (getNextChar('f', 'F') >= 0) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } return TokenNameFloatingPointLiteral; } if (getNextChar('d', 'D') >= 0) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } return TokenNameDoubleLiteral; } if (getNextChar('l', 'L') >= 0) { if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } throw new InvalidInputException(INVALID_HEXA); } if (this.sourceLevel < ClassFileConstants.JDK1_5) { throw new InvalidInputException(ILLEGAL_HEXA_LITERAL); } return TokenNameDoubleLiteral; } else { if (end == start) throw new InvalidInputException(INVALID_HEXA); return TokenNameIntegerLiteral; } } else if (getNextChar('b', 'B') >= 0) { //----------binary----------------- int start = this.currentPosition; consumeDigits(2, true); int end = this.currentPosition; if (end == start) { if (this.sourceLevel < ClassFileConstants.JDK1_7) { throw new InvalidInputException(BINARY_LITERAL_NOT_BELOW_17); } throw new InvalidInputException(INVALID_BINARY); } if (getNextChar('l', 'L') >= 0) { if (this.sourceLevel < ClassFileConstants.JDK1_7) { throw new InvalidInputException(BINARY_LITERAL_NOT_BELOW_17); } return TokenNameLongLiteral; } if (this.sourceLevel < ClassFileConstants.JDK1_7) { throw new InvalidInputException(BINARY_LITERAL_NOT_BELOW_17); } return TokenNameIntegerLiteral; } //there is no x or X nor b or B in the number //potential octal if (getNextCharAsDigit()) { //-------------potential octal----------------- consumeDigits(10); if (getNextChar('l', 'L') >= 0) { return TokenNameLongLiteral; } if (getNextChar('f', 'F') >= 0) { return TokenNameFloatingPointLiteral; } if (getNextChar('d', 'D') >= 0) { return TokenNameDoubleLiteral; } else { //make the distinction between octal and float .... boolean isInteger = true; if (getNextChar('.')) { isInteger = false; consumeDigits(10); } if (getNextChar('e', 'E') >= 0) { // consume next character isInteger = false; this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } if ((this.currentCharacter == '-') || (this.currentCharacter == '+')) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } if (!ScannerHelper.isDigit(this.currentCharacter)) { if (this.currentCharacter == '_') { // wrongly place '_' consumeDigits(10); throw new InvalidInputException(INVALID_UNDERSCORE); } throw new InvalidInputException(INVALID_FLOAT); } consumeDigits(10); } if (getNextChar('f', 'F') >= 0) return TokenNameFloatingPointLiteral; if (getNextChar('d', 'D') >= 0 || !isInteger) return TokenNameDoubleLiteral; return TokenNameIntegerLiteral; } } else { /* carry on */ } } consumeDigits(10); if ((!dotPrefix) && (getNextChar('l', 'L') >= 0)) return TokenNameLongLiteral; if ((!dotPrefix) && (getNextChar('.'))) { //decimal part that can be empty consumeDigits(10, true); floating = true; } //if floating is true both exponant and suffix may be optional if (getNextChar('e', 'E') >= 0) { floating = true; // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } if ((this.currentCharacter == '-') || (this.currentCharacter == '+')) { // consume next character this.unicodeAsBackSlash = false; if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) { getNextUnicodeChar(); } else { if (this.withoutUnicodePtr != 0) { unicodeStore(); } } } if (!ScannerHelper.isDigit(this.currentCharacter)) { if (this.currentCharacter == '_') { // wrongly place '_' consumeDigits(10); throw new InvalidInputException(INVALID_UNDERSCORE); } throw new InvalidInputException(INVALID_FLOAT); } // current character is a digit so we expect no digit first (the next character could be an underscore) consumeDigits(10); } if (getNextChar('d', 'D') >= 0) return TokenNameDoubleLiteral; if (getNextChar('f', 'F') >= 0) return TokenNameFloatingPointLiteral; //the long flag has been tested before return floating ? TokenNameDoubleLiteral : TokenNameIntegerLiteral; }
Search the line number corresponding to a specific position
Params:
  • position – int
Returns:int
/** * Search the line number corresponding to a specific position * @param position int * @return int */
public final int getLineNumber(int position) { return Util.getLineNumber(position, this.lineEnds, 0, this.linePtr); } public final void setSource(char[] sourceString){ //the source-buffer is set to sourceString int sourceLength; if (sourceString == null) { this.source = CharOperation.NO_CHAR; sourceLength = 0; } else { this.source = sourceString; sourceLength = sourceString.length; } this.startPosition = -1; this.eofPosition = sourceLength; this.initialPosition = this.currentPosition = 0; this.containsAssertKeyword = false; this.linePtr = -1; this.scanContext = null; this.yieldColons = -1; this.insideModuleInfo = false; } /* * Should be used if a parse (usually a diet parse) has already been performed on the unit, * so as to get the already computed line end positions. */ public final void setSource(char[] contents, CompilationResult compilationResult) { if (contents == null) { char[] cuContents = compilationResult.compilationUnit.getContents(); setSource(cuContents); } else { setSource(contents); } int[] lineSeparatorPositions = compilationResult.lineSeparatorPositions; if (lineSeparatorPositions != null) { this.lineEnds = lineSeparatorPositions; this.linePtr = lineSeparatorPositions.length - 1; } } /* * Should be used if a parse (usually a diet parse) has already been performed on the unit, * so as to get the already computed line end positions. */ public final void setSource(CompilationResult compilationResult) { setSource(null, compilationResult); } @Override public String toString() { if (this.startPosition == this.eofPosition) return "EOF\n\n" + new String(this.source); //$NON-NLS-1$ if (this.currentPosition > this.eofPosition) return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$ if (this.currentPosition <= 0) return "NOT started!\n\n"+ (this.source != null ? new String(this.source) : ""); //$NON-NLS-1$ //$NON-NLS-2$ StringBuffer buffer = new StringBuffer(); if (this.startPosition < 1000) { buffer.append(this.source, 0, this.startPosition); } else { buffer.append("<source beginning>\n...\n"); //$NON-NLS-1$ int line = Util.getLineNumber(this.startPosition-1000, this.lineEnds, 0, this.linePtr); int lineStart = getLineStart(line); buffer.append(this.source, lineStart, this.startPosition-lineStart); } buffer.append("\n===============================\nStarts here -->"); //$NON-NLS-1$ int middleLength = (this.currentPosition - 1) - this.startPosition + 1; if (middleLength > -1) { buffer.append(this.source, this.startPosition, middleLength); } if (this.nextToken != TerminalTokens.TokenNameNotAToken) { buffer.append("<-- Ends here [in pipeline " + toStringAction(this.nextToken) + "]\n===============================\n"); //$NON-NLS-1$ //$NON-NLS-2$ } else { buffer.append("<-- Ends here\n===============================\n"); //$NON-NLS-1$ } buffer.append(this.source, (this.currentPosition - 1) + 1, this.eofPosition - (this.currentPosition - 1) - 1); return buffer.toString(); } public String toStringAction(int act) { switch (act) { case TokenNameIdentifier : return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameRestrictedIdentifierYield : return "yield"; //$NON-NLS-1$ case TokenNameabstract : return "abstract"; //$NON-NLS-1$ case TokenNameboolean : return "boolean"; //$NON-NLS-1$ case TokenNamebreak : return "break"; //$NON-NLS-1$ case TokenNamebyte : return "byte"; //$NON-NLS-1$ case TokenNamecase : return "case"; //$NON-NLS-1$ case TokenNamecatch : return "catch"; //$NON-NLS-1$ case TokenNamechar : return "char"; //$NON-NLS-1$ case TokenNameclass : return "class"; //$NON-NLS-1$ case TokenNamecontinue : return "continue"; //$NON-NLS-1$ case TokenNamedefault : return "default"; //$NON-NLS-1$ case TokenNamedo : return "do"; //$NON-NLS-1$ case TokenNamedouble : return "double"; //$NON-NLS-1$ case TokenNameelse : return "else"; //$NON-NLS-1$ case TokenNameextends : return "extends"; //$NON-NLS-1$ case TokenNamefalse : return "false"; //$NON-NLS-1$ case TokenNamefinal : return "final"; //$NON-NLS-1$ case TokenNamefinally : return "finally"; //$NON-NLS-1$ case TokenNamefloat : return "float"; //$NON-NLS-1$ case TokenNamefor : return "for"; //$NON-NLS-1$ case TokenNameif : return "if"; //$NON-NLS-1$ case TokenNameimplements : return "implements"; //$NON-NLS-1$ case TokenNameimport : return "import"; //$NON-NLS-1$ case TokenNameinstanceof : return "instanceof"; //$NON-NLS-1$ case TokenNameint : return "int"; //$NON-NLS-1$ case TokenNameinterface : return "interface"; //$NON-NLS-1$ case TokenNamelong : return "long"; //$NON-NLS-1$ case TokenNamenative : return "native"; //$NON-NLS-1$ case TokenNamenew : return "new"; //$NON-NLS-1$ case TokenNamenull : return "null"; //$NON-NLS-1$ case TokenNamepackage : return "package"; //$NON-NLS-1$ case TokenNameprivate : return "private"; //$NON-NLS-1$ case TokenNameprotected : return "protected"; //$NON-NLS-1$ case TokenNamepublic : return "public"; //$NON-NLS-1$ case TokenNamereturn : return "return"; //$NON-NLS-1$ case TokenNameshort : return "short"; //$NON-NLS-1$ case TokenNamestatic : return "static"; //$NON-NLS-1$ case TokenNamesuper : return "super"; //$NON-NLS-1$ case TokenNameswitch : return "switch"; //$NON-NLS-1$ case TokenNamesynchronized : return "synchronized"; //$NON-NLS-1$ case TokenNamethis : return "this"; //$NON-NLS-1$ case TokenNamethrow : return "throw"; //$NON-NLS-1$ case TokenNamethrows : return "throws"; //$NON-NLS-1$ case TokenNametransient : return "transient"; //$NON-NLS-1$ case TokenNametrue : return "true"; //$NON-NLS-1$ case TokenNametry : return "try"; //$NON-NLS-1$ case TokenNamevoid : return "void"; //$NON-NLS-1$ case TokenNamevolatile : return "volatile"; //$NON-NLS-1$ case TokenNamewhile : return "while"; //$NON-NLS-1$ case TokenNamemodule : return "module"; //$NON-NLS-1$ case TokenNamerequires : return "requires"; //$NON-NLS-1$ case TokenNameexports : return "exports"; //$NON-NLS-1$ case TokenNameIntegerLiteral : return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameLongLiteral : return "Long(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameFloatingPointLiteral : return "Float(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameDoubleLiteral : return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameCharacterLiteral : return "Char(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameStringLiteral : return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameTextBlock : return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNamePLUS_PLUS : return "++"; //$NON-NLS-1$ case TokenNameMINUS_MINUS : return "--"; //$NON-NLS-1$ case TokenNameEQUAL_EQUAL : return "=="; //$NON-NLS-1$ case TokenNameLESS_EQUAL : return "<="; //$NON-NLS-1$ case TokenNameGREATER_EQUAL : return ">="; //$NON-NLS-1$ case TokenNameNOT_EQUAL : return "!="; //$NON-NLS-1$ case TokenNameLEFT_SHIFT : return "<<"; //$NON-NLS-1$ case TokenNameRIGHT_SHIFT : return ">>"; //$NON-NLS-1$ case TokenNameUNSIGNED_RIGHT_SHIFT : return ">>>"; //$NON-NLS-1$ case TokenNamePLUS_EQUAL : return "+="; //$NON-NLS-1$ case TokenNameMINUS_EQUAL : return "-="; //$NON-NLS-1$ case TokenNameARROW : return "->"; //$NON-NLS-1$ case TokenNameMULTIPLY_EQUAL : return "*="; //$NON-NLS-1$ case TokenNameDIVIDE_EQUAL : return "/="; //$NON-NLS-1$ case TokenNameAND_EQUAL : return "&="; //$NON-NLS-1$ case TokenNameOR_EQUAL : return "|="; //$NON-NLS-1$ case TokenNameXOR_EQUAL : return "^="; //$NON-NLS-1$ case TokenNameREMAINDER_EQUAL : return "%="; //$NON-NLS-1$ case TokenNameLEFT_SHIFT_EQUAL : return "<<="; //$NON-NLS-1$ case TokenNameRIGHT_SHIFT_EQUAL : return ">>="; //$NON-NLS-1$ case TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL : return ">>>="; //$NON-NLS-1$ case TokenNameOR_OR : return "||"; //$NON-NLS-1$ case TokenNameAND_AND : return "&&"; //$NON-NLS-1$ case TokenNamePLUS : return "+"; //$NON-NLS-1$ case TokenNameMINUS : return "-"; //$NON-NLS-1$ case TokenNameNOT : return "!"; //$NON-NLS-1$ case TokenNameREMAINDER : return "%"; //$NON-NLS-1$ case TokenNameXOR : return "^"; //$NON-NLS-1$ case TokenNameAND : return "&"; //$NON-NLS-1$ case TokenNameMULTIPLY : return "*"; //$NON-NLS-1$ case TokenNameOR : return "|"; //$NON-NLS-1$ case TokenNameTWIDDLE : return "~"; //$NON-NLS-1$ case TokenNameDIVIDE : return "/"; //$NON-NLS-1$ case TokenNameGREATER : return ">"; //$NON-NLS-1$ case TokenNameLESS : return "<"; //$NON-NLS-1$ case TokenNameLPAREN : return "("; //$NON-NLS-1$ case TokenNameRPAREN : return ")"; //$NON-NLS-1$ case TokenNameLBRACE : return "{"; //$NON-NLS-1$ case TokenNameRBRACE : return "}"; //$NON-NLS-1$ case TokenNameLBRACKET : return "["; //$NON-NLS-1$ case TokenNameRBRACKET : return "]"; //$NON-NLS-1$ case TokenNameSEMICOLON : return ";"; //$NON-NLS-1$ case TokenNameQUESTION : return "?"; //$NON-NLS-1$ case TokenNameCOLON : return ":"; //$NON-NLS-1$ case TokenNameCOLON_COLON : return "::"; //$NON-NLS-1$ case TokenNameCOMMA : return ","; //$NON-NLS-1$ case TokenNameDOT : return "."; //$NON-NLS-1$ case TokenNameEQUAL : return "="; //$NON-NLS-1$ case TokenNameEOF : return "EOF"; //$NON-NLS-1$ case TokenNameWHITESPACE : return "white_space(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ default : return "not-a-token"; //$NON-NLS-1$ } } public void unicodeInitializeBuffer(int length) { this.withoutUnicodePtr = length; if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[length+(1+10)]; int bLength = this.withoutUnicodeBuffer.length; if (1+length >= bLength) { System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length + (1+10)], 0, bLength); } System.arraycopy(this.source, this.startPosition, this.withoutUnicodeBuffer, 1, length); } public void unicodeStore() { int pos = ++this.withoutUnicodePtr; if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[10]; int length = this.withoutUnicodeBuffer.length; if (pos == length) { System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length * 2], 0, length); } this.withoutUnicodeBuffer[pos] = this.currentCharacter; } public void unicodeStore(char character) { int pos = ++this.withoutUnicodePtr; if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[10]; int length = this.withoutUnicodeBuffer.length; if (pos == length) { System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length * 2], 0, length); } this.withoutUnicodeBuffer[pos] = character; } public static boolean isIdentifier(int token) { return token == TerminalTokens.TokenNameIdentifier; } public static boolean isLiteral(int token) { switch(token) { case TerminalTokens.TokenNameIntegerLiteral: case TerminalTokens.TokenNameLongLiteral: case TerminalTokens.TokenNameFloatingPointLiteral: case TerminalTokens.TokenNameDoubleLiteral: case TerminalTokens.TokenNameStringLiteral: case TerminalTokens.TokenNameTextBlock: case TerminalTokens.TokenNameCharacterLiteral: return true; default: return false; } } public static boolean isKeyword(int token) { switch(token) { case TerminalTokens.TokenNameabstract: case TerminalTokens.TokenNameassert: case TerminalTokens.TokenNamebyte: case TerminalTokens.TokenNamebreak: case TerminalTokens.TokenNameboolean: case TerminalTokens.TokenNamecase: case TerminalTokens.TokenNamechar: case TerminalTokens.TokenNamecatch: case TerminalTokens.TokenNameclass: case TerminalTokens.TokenNamecontinue: case TerminalTokens.TokenNamedo: case TerminalTokens.TokenNamedouble: case TerminalTokens.TokenNamedefault: case TerminalTokens.TokenNameelse: case TerminalTokens.TokenNameextends: case TerminalTokens.TokenNamefor: case TerminalTokens.TokenNamefinal: case TerminalTokens.TokenNamefloat: case TerminalTokens.TokenNamefalse: case TerminalTokens.TokenNamefinally: case TerminalTokens.TokenNameif: case TerminalTokens.TokenNameint: case TerminalTokens.TokenNameimport: case TerminalTokens.TokenNameinterface: case TerminalTokens.TokenNameimplements: case TerminalTokens.TokenNameinstanceof: case TerminalTokens.TokenNamelong: case TerminalTokens.TokenNamenew: case TerminalTokens.TokenNamenull: case TerminalTokens.TokenNamenative: case TerminalTokens.TokenNamepublic: case TerminalTokens.TokenNamepackage: case TerminalTokens.TokenNameprivate: case TerminalTokens.TokenNameprotected: case TerminalTokens.TokenNamereturn: case TerminalTokens.TokenNameshort: case TerminalTokens.TokenNamesuper: case TerminalTokens.TokenNamestatic: case TerminalTokens.TokenNameswitch: case TerminalTokens.TokenNamestrictfp: case TerminalTokens.TokenNamesynchronized: case TerminalTokens.TokenNametry: case TerminalTokens.TokenNamethis: case TerminalTokens.TokenNametrue: case TerminalTokens.TokenNamethrow: case TerminalTokens.TokenNamethrows: case TerminalTokens.TokenNametransient: case TerminalTokens.TokenNamevoid: case TerminalTokens.TokenNamevolatile: case TerminalTokens.TokenNamewhile: return true; case TerminalTokens.TokenNameRestrictedIdentifierYield: // making explicit - yield not a (restricted) keyword but restricted identifier. //$FALL-THROUGH$ default: return false; } } // Vanguard Scanner - A Private utility helper class for the scanner. private static final class VanguardScanner extends Scanner { public VanguardScanner(long sourceLevel, long complianceLevel, boolean previewEnabled) { super (false /*comment*/, false /*whitespace*/, false /*nls*/, sourceLevel, complianceLevel, null/*taskTag*/, null/*taskPriorities*/, false /*taskCaseSensitive*/, previewEnabled); } @Override public int getNextToken() throws InvalidInputException { int token; if (this.nextToken != TokenNameNotAToken) { token = this.nextToken; this.nextToken = TokenNameNotAToken; return token; // presumed to be unambiguous. } if (this.scanContext == null) { // init lazily, since isInModuleDeclaration may need the parser to be known this.scanContext = isInModuleDeclaration() ? ScanContext.EXPECTING_KEYWORD : ScanContext.INACTIVE; } token = getNextToken0(); if (areRestrictedModuleKeywordsActive()) { if (isRestrictedKeyword(token)) token = disambiguatedRestrictedKeyword(token); updateScanContext(token); } if (token == TokenNameAT && atTypeAnnotation()) { if (((VanguardParser) this.activeParser).currentGoal == Goal.LambdaParameterListGoal) { token = disambiguatedToken(token); } else { token = TokenNameAT308; } } return token == TokenNameEOF ? TokenNameNotAToken : token; } } private static class Goal { int first; // steer the parser towards a single minded pursuit. int [] follow; // the definite terminal symbols that signal the successful reduction to goal. int rule; static int LambdaParameterListRule = 0; static int IntersectionCastRule = 0; static int ReferenceExpressionRule = 0; static int VarargTypeAnnotationsRule = 0; static int BlockStatementoptRule = 0; static int YieldStatementRule = 0; static Goal LambdaParameterListGoal; static Goal IntersectionCastGoal; static Goal VarargTypeAnnotationGoal; static Goal ReferenceExpressionGoal; static Goal BlockStatementoptGoal; static Goal YieldStatementGoal; static { for (int i = 1; i <= ParserBasicInformation.NUM_RULES; i++) { // 0 == $acc if ("ParenthesizedLambdaParameterList".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$ LambdaParameterListRule = i; else if ("ParenthesizedCastNameAndBounds".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$ IntersectionCastRule = i; else if ("ReferenceExpressionTypeArgumentsAndTrunk".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$ ReferenceExpressionRule = i; else if ("TypeAnnotations".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$ VarargTypeAnnotationsRule = i; else if ("BlockStatementopt".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$ BlockStatementoptRule = i; else if ("YieldStatement".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$ YieldStatementRule = i; } LambdaParameterListGoal = new Goal(TokenNameARROW, new int[] { TokenNameARROW }, LambdaParameterListRule); IntersectionCastGoal = new Goal(TokenNameLPAREN, followSetOfCast(), IntersectionCastRule); VarargTypeAnnotationGoal = new Goal(TokenNameAT, new int[] { TokenNameELLIPSIS }, VarargTypeAnnotationsRule); ReferenceExpressionGoal = new Goal(TokenNameLESS, new int[] { TokenNameCOLON_COLON }, ReferenceExpressionRule); BlockStatementoptGoal = new Goal(TokenNameLBRACE, new int [0], BlockStatementoptRule); YieldStatementGoal = new Goal(TokenNameARROW, new int [0], YieldStatementRule); } Goal(int first, int [] follow, int rule) { this.first = first; this.follow = follow; this.rule = rule; } boolean hasBeenReached(int act, int token) { /* System.out.println("[Goal = " + Parser.name[Parser.non_terminal_index[Parser.lhs[this.rule]]] + "] " + "Saw: " + Parser.name[Parser.non_terminal_index[Parser.lhs[act]]] + "::" + //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ Parser.name[Parser.terminal_index[token]]); */ if (act == this.rule) { final int length = this.follow.length; if (length == 0) return true; for (int i = 0; i < length; i++) if (this.follow[i] == token) return true; } return false; } private static int [] followSetOfCast() { return new int [] { TokenNameIdentifier, TokenNamenew, TokenNamesuper, TokenNamethis, TokenNamefalse, TokenNametrue, TokenNamenull, TokenNameIntegerLiteral, TokenNameLongLiteral, TokenNameFloatingPointLiteral, TokenNameDoubleLiteral, TokenNameCharacterLiteral, TokenNameStringLiteral, TokenNameTextBlock, TokenNameNOT, TokenNameTWIDDLE, TokenNameLPAREN }; } } // Vanguard Parser - A Private utility helper class for the scanner. private static class VanguardParser extends Parser { public static final boolean SUCCESS = true; public static final boolean FAILURE = false; Goal currentGoal; public VanguardParser(VanguardScanner scanner) { this.scanner = scanner; } public VanguardParser(ProblemReporter reporter) { super(reporter, false); } // Canonical LALR pushdown automaton identical to Parser.parse() minus side effects of any kind, returns the rule reduced. protected boolean parse(Goal goal) { this.currentGoal = goal; try { int act = START_STATE; this.stateStackTop = -1; this.currentToken = goal.first; ProcessTerminals : for (;;) { int stackLength = this.stack.length; if (++this.stateStackTop >= stackLength) { System.arraycopy( this.stack, 0, this.stack = new int[stackLength + StackIncrement], 0, stackLength); } this.stack[this.stateStackTop] = act; act = Parser.tAction(act, this.currentToken); if (act == ERROR_ACTION) { return FAILURE; } if (act <= NUM_RULES) { this.stateStackTop--; } else if (act > ERROR_ACTION) { /* shift-reduce */ this.unstackedAct = act; try { this.currentToken = this.scanner.getNextToken(); } finally { this.unstackedAct = ERROR_ACTION; } act -= ERROR_ACTION; } else { if (act < ACCEPT_ACTION) { /* shift */ this.unstackedAct = act; try { this.currentToken = this.scanner.getNextToken(); } finally { this.unstackedAct = ERROR_ACTION; } continue ProcessTerminals; } return FAILURE; // accept - we should never reach this state, we accept at reduce with a right member of follow set below. } // ProcessNonTerminals : do { /* reduce */ if (goal.hasBeenReached(act, this.currentToken)) return SUCCESS; this.stateStackTop -= (Parser.rhs[act] - 1); act = Parser.ntAction(this.stack[this.stateStackTop], Parser.lhs[act]); } while (act <= NUM_RULES); } } catch (Exception e) { return FAILURE; } } @Override public String toString() { return "\n\n\n----------------Scanner--------------\n" + this.scanner.toString(); //$NON-NLS-1$; } } private class ScanContextDetector extends VanguardParser { ScanContextDetector(CompilerOptions options) { super(new ProblemReporter( DefaultErrorHandlingPolicies.ignoreAllProblems(), options, new DefaultProblemFactory())); this.problemReporter.options.performStatementsRecovery = false; this.reportSyntaxErrorIsRequired = false; this.reportOnlyOneSyntaxError = false; } @Override public void initializeScanner(){ this.scanner = new Scanner( false /*comment*/, false /*whitespace*/, false, /* will be set in initialize(boolean) */ this.options.sourceLevel /*sourceLevel*/, this.options.complianceLevel /*complianceLevel*/, this.options.taskTags/*taskTags*/, this.options.taskPriorities/*taskPriorities*/, this.options.isTaskCaseSensitive/*taskCaseSensitive*/, this.options.enablePreviewFeatures /*isPreviewEnabled*/) { @Override void updateScanContext(int token) { if (token != TokenNameEOF) super.updateScanContext(token); } }; this.scanner.recordLineSeparator = false; this.scanner.setActiveParser(this); this.scanner.previewEnabled = this.options.enablePreviewFeatures; } @Override public boolean isParsingModuleDeclaration() { return true; } public ScanContext getScanContext(char[] src, int begin) { this.scanner.setSource(src); this.scanner.resetTo(0, begin); goForCompilationUnit(); Goal goal = new Goal(TokenNamePLUS_PLUS, null, 0) { @Override boolean hasBeenReached(int act, int token) { return token == TokenNameEOF; } }; parse(goal); return this.scanner.scanContext; } } private VanguardParser getVanguardParser() { if (this.vanguardParser == null) { this.vanguardScanner = new VanguardScanner(this.sourceLevel, this.complianceLevel, this.previewEnabled); this.vanguardParser = new VanguardParser(this.vanguardScanner); this.vanguardScanner.setActiveParser(this.vanguardParser); } this.vanguardScanner.setSource(this.source); this.vanguardScanner.resetTo(this.startPosition, this.eofPosition - 1, isInModuleDeclaration(), this.scanContext); return this.vanguardParser; } protected final boolean mayBeAtBreakPreview() { return this.breakPreviewAllowed && this.lookBack[1] != TokenNameARROW; } protected final boolean maybeAtLambdaOrCast() { // Could the '(' we saw just now herald a lambda parameter list or a cast expression ? (the possible locations for both are identical.) switch (this.lookBack[1]) { case TokenNameIdentifier: case TokenNamecatch: case TokenNamethis: case TokenNamesuper: case TokenNameif: case TokenNameswitch: case TokenNamewhile: case TokenNamefor: case TokenNamesynchronized: case TokenNametry: return false; // not a viable prefix for cast or lambda. default: return this.activeParser.atConflictScenario(TokenNameLPAREN); } } protected final boolean maybeAtReferenceExpression() { // Did the '<' we saw just now herald a reference expression's type arguments and trunk ? switch (this.lookBack[1]) { case TokenNameIdentifier: switch (this.lookBack[0]) { case TokenNameSEMICOLON: // for (int i = 0; i < 10; i++); case TokenNameRBRACE: // class X { void foo() {} X<String> x = null; } case TokenNameclass: // class X<T> {} case TokenNameinterface: // interface I<T> {} case TokenNameenum: // enum E<T> {} case TokenNamefinal: // final Collection<String> case TokenNameLESS: // Collection<IScalarData<AbstractData>> case TokenNameGREATER: // public <T> List<T> foo() { /* */ } case TokenNameRIGHT_SHIFT:// static <T extends SelfType<T>> List<T> makeSingletonList(T t) { /* */ } case TokenNamenew: // new ArrayList<String>(); case TokenNamepublic: // public List<String> foo() {} case TokenNameabstract: // abstract List<String> foo() {} case TokenNameprivate: // private List<String> foo() {} case TokenNameprotected: // protected List<String> foo() {} case TokenNamestatic: // public static List<String> foo() {} case TokenNameextends: // <T extends Y<Z>> case TokenNamesuper: // ? super Context<N> case TokenNameAND: // T extends Object & Comparable<? super T> case TokenNameimplements: // class A implements I<Z> case TokenNamethrows: // throws Y<Z> case TokenNameAT: // @Deprecated <T> void foo() {} case TokenNameinstanceof: // if (o instanceof List<E>[]) return false; default: break; } break; case TokenNameNotAToken: // Not kosher, don't touch. break; default: return false; } return this.activeParser.atConflictScenario(TokenNameLESS); } private final boolean maybeAtEllipsisAnnotationsStart() { // Did the '@' we saw just now herald a type annotation on a ... ? Presumed to be at type annotation already. if (this.consumingEllipsisAnnotations) return false; switch (this.lookBack[1]) { case TokenNamenew: case TokenNameCOMMA: case TokenNameextends: case TokenNamesuper: case TokenNameimplements: case TokenNameDOT: case TokenNameLBRACE: case TokenNameinstanceof: case TokenNameLESS: case TokenNameAND: case TokenNamethrows: return false; default: return true; } } protected final boolean atTypeAnnotation() { // Did the '@' we saw just now herald a type annotation ? We should not ask the parser whether it would shift @308 ! return !this.activeParser.atConflictScenario(TokenNameAT); } public void setActiveParser(ConflictedParser parser) { this.activeParser = parser; this.lookBack[0] = this.lookBack[1] = TokenNameNotAToken; // no hand me downs please. if (parser != null) { this.insideModuleInfo = parser.isParsingModuleDeclaration(); } } public static boolean isRestrictedKeyword(int token) { switch(token) { case TokenNameopen: case TokenNamemodule: case TokenNamerequires: case TokenNametransitive: case TokenNameexports: case TokenNameto: case TokenNameopens: case TokenNameuses: case TokenNameprovides: case TokenNamewith: return true; default: return false; } } private boolean mayBeAtAnYieldStatement() { // preceded by ;, {, }, ), or -> [Ref: http://mail.openjdk.java.net/pipermail/amber-spec-experts/2019-May/001401.html] // above comment is super-seded by http://mail.openjdk.java.net/pipermail/amber-spec-experts/2019-May/001414.html switch (this.lookBack[1]) { case TokenNameLBRACE: case TokenNameRBRACE: case TokenNameRPAREN: case TokenNameSEMICOLON: case TokenNameelse: case TokenNamedo: return true; case TokenNameCOLON: return this.lookBack[0] == TokenNamedefault || this.yieldColons == 1; case TokenNameDOT: case TokenNameARROW: default: return false; } } private boolean disambiguateYieldWithLookAhead() { getVanguardParser(); this.vanguardScanner.resetTo(this.currentPosition, this.eofPosition - 1); try { int lookAhead1 = this.vanguardScanner.getNextToken(); switch (lookAhead1) { case TokenNameEQUAL_EQUAL : case TokenNameLESS_EQUAL : case TokenNameGREATER_EQUAL : case TokenNameNOT_EQUAL : case TokenNameLEFT_SHIFT : case TokenNameRIGHT_SHIFT : case TokenNameUNSIGNED_RIGHT_SHIFT : case TokenNamePLUS_EQUAL : case TokenNameMINUS_EQUAL : case TokenNameMULTIPLY_EQUAL : case TokenNameDIVIDE_EQUAL : case TokenNameAND_EQUAL : case TokenNameOR_EQUAL : case TokenNameXOR_EQUAL : case TokenNameREMAINDER_EQUAL : case TokenNameLEFT_SHIFT_EQUAL : case TokenNameRIGHT_SHIFT_EQUAL : case TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL : case TokenNameOR_OR : case TokenNameAND_AND : case TokenNameREMAINDER : case TokenNameXOR : case TokenNameAND : case TokenNameMULTIPLY : case TokenNameOR : case TokenNameTWIDDLE : case TokenNameDIVIDE : case TokenNameGREATER : case TokenNameLESS : case TokenNameLBRACE : case TokenNameRBRACE : case TokenNameLBRACKET : case TokenNameRBRACKET : case TokenNameSEMICOLON : case TokenNameQUESTION : case TokenNameCOLON : case TokenNameCOMMA : case TokenNameDOT : case TokenNameEQUAL : case TokenNameAT : case TokenNameELLIPSIS : case TokenNameARROW : case TokenNameCOLON_COLON : return false; case TokenNameMINUS_MINUS : case TokenNamePLUS_PLUS : int lookAhead2 = this.vanguardScanner.getNextToken(); return lookAhead2 == TokenNameIdentifier; default : return true; } } catch (InvalidInputException e) { if (e.getMessage().equals(INVALID_CHAR_IN_STRING)) { //Ignore } else { // Shouldn't happen, but log the error e.printStackTrace(); } } return false; // IIE event; } int disambiguatedRestrictedIdentifierYield(int restrictedIdentifierToken) { // and here's the kludge if (restrictedIdentifierToken != TokenNameRestrictedIdentifierYield) return restrictedIdentifierToken; if (this.sourceLevel < ClassFileConstants.JDK13 || !this.previewEnabled) return TokenNameIdentifier; return mayBeAtAnYieldStatement() && disambiguateYieldWithLookAhead() ? restrictedIdentifierToken : TokenNameIdentifier; } int disambiguatedRestrictedKeyword(int restrictedKeywordToken) { int token = restrictedKeywordToken; if (this.scanContext == ScanContext.EXPECTING_IDENTIFIER) return TokenNameIdentifier; switch(restrictedKeywordToken) { case TokenNametransitive: if (this.scanContext != ScanContext.AFTER_REQUIRES) { token = TokenNameIdentifier; } else { getVanguardParser(); this.vanguardScanner.resetTo(this.currentPosition, this.eofPosition - 1, true, ScanContext.EXPECTING_IDENTIFIER); try { int lookAhead = this.vanguardScanner.getNextToken(); if (lookAhead == TokenNameSEMICOLON) token = TokenNameIdentifier; } catch (InvalidInputException e) { // } } break; case TokenNameopen: case TokenNamemodule: case TokenNameexports: case TokenNameopens: case TokenNamerequires: case TokenNameprovides: case TokenNameuses: case TokenNameto: case TokenNamewith: if (this.scanContext != ScanContext.EXPECTING_KEYWORD) { token = TokenNameIdentifier; } break; } return token; } int disambiguatedToken(int token) { final VanguardParser parser = getVanguardParser(); if (token == TokenNameARROW && this.inCase) { this.nextToken = TokenNameARROW; this.inCase = false; return TokenNameBeginCaseExpr; } else if (token == TokenNameLPAREN && maybeAtLambdaOrCast()) { if (parser.parse(Goal.LambdaParameterListGoal) == VanguardParser.SUCCESS) { this.nextToken = TokenNameLPAREN; return TokenNameBeginLambda; } this.vanguardScanner.resetTo(this.startPosition, this.eofPosition - 1); if (parser.parse(Goal.IntersectionCastGoal) == VanguardParser.SUCCESS) { this.nextToken = TokenNameLPAREN; return TokenNameBeginIntersectionCast; } } else if (token == TokenNameLESS && maybeAtReferenceExpression()) { if (parser.parse(Goal.ReferenceExpressionGoal) == VanguardParser.SUCCESS) { this.nextToken = TokenNameLESS; return TokenNameBeginTypeArguments; } } else if (token == TokenNameAT && atTypeAnnotation()) { token = TokenNameAT308; if (maybeAtEllipsisAnnotationsStart()) { if (parser.parse(Goal.VarargTypeAnnotationGoal) == VanguardParser.SUCCESS) { this.consumingEllipsisAnnotations = true; this.nextToken = TokenNameAT308; return TokenNameAT308DOTDOTDOT; } } } return token; } protected boolean isAtAssistIdentifier() { return false; } // Position the scanner at the next block statement and return the start token. We recognize empty statements. public int fastForward(Statement unused) { int token; while (true) { try { token = getNextToken(); } catch (InvalidInputException e) { return TokenNameEOF; } /* FOLLOW map of BlockStatement, since the non-terminal is recursive is a super set of its own FIRST set. We use FOLLOW rather than FIRST since we want to recognize empty statements. i.e if (x > 10) { x = 0 } */ switch(token) { case TokenNameIdentifier: if (isAtAssistIdentifier()) // do not fast forward past the assist identifier ! We don't handle collections as of now. return token; //$FALL-THROUGH$ case TokenNameabstract: case TokenNameassert: case TokenNameboolean: case TokenNamebreak: case TokenNamebyte: case TokenNamecase: case TokenNamechar: case TokenNameclass: case TokenNamecontinue: case TokenNamedefault: case TokenNamedo: case TokenNamedouble: case TokenNameenum: case TokenNamefalse: case TokenNamefinal: case TokenNamefloat: case TokenNamefor: case TokenNameif: case TokenNameint: case TokenNameinterface: case TokenNamelong: case TokenNamenative: case TokenNamenew: case TokenNamenull: case TokenNameprivate: case TokenNameprotected: case TokenNamepublic: case TokenNamereturn: case TokenNameshort: case TokenNamestatic: case TokenNamestrictfp: case TokenNamesuper: case TokenNameswitch: case TokenNamesynchronized: case TokenNamethis: case TokenNamethrow: case TokenNametransient: case TokenNametrue: case TokenNametry: case TokenNamevoid: case TokenNamevolatile: case TokenNamewhile: case TokenNameIntegerLiteral: // ??! case TokenNameLongLiteral: case TokenNameFloatingPointLiteral: case TokenNameDoubleLiteral: case TokenNameCharacterLiteral: case TokenNameStringLiteral: case TokenNameTextBlock: case TokenNamePLUS_PLUS: case TokenNameMINUS_MINUS: case TokenNameLESS: case TokenNameLPAREN: case TokenNameLBRACE: case TokenNameAT: case TokenNameBeginLambda: case TokenNameAT308: case TokenNameRestrictedIdentifierYield: // can be in FOLLOW of Block if(getVanguardParser().parse(Goal.BlockStatementoptGoal) == VanguardParser.SUCCESS) return token; break; case TokenNameSEMICOLON: case TokenNameEOF: return token; case TokenNameRBRACE: // simulate empty statement. ungetToken(token); return TokenNameSEMICOLON; default: break; } } }
Overridable hook, to allow CompletionScanner to hide a faked identifier token.
/** Overridable hook, to allow CompletionScanner to hide a faked identifier token. */
protected int getNextNotFakedToken() throws InvalidInputException { return getNextToken(); } }