/*
 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
 * Use of this file is governed by the BSD 3-clause license that
 * can be found in the LICENSE.txt file in the project root.
 */

package org.antlr.v4.runtime;

This class extends BufferedTokenStream with functionality to filter token streams to tokens on a particular channel (tokens where Token.getChannel returns a particular value).

This token stream provides access to all tokens by index or when calling methods like BufferedTokenStream.getText. The channel filtering is only used for code accessing tokens via the lookahead methods BufferedTokenStream.LA, LT, and LB.

By default, tokens are placed on the default channel (Token.DEFAULT_CHANNEL), but may be reassigned by using the ->channel(HIDDEN) lexer command, or by using an embedded action to call Lexer.setChannel.

Note: lexer rules which use the ->skip lexer command or call Lexer.skip do not produce tokens at all, so input text matched by such a rule will not be available as part of the token stream, regardless of channel.

we
/** * This class extends {@link BufferedTokenStream} with functionality to filter * token streams to tokens on a particular channel (tokens where * {@link Token#getChannel} returns a particular value). * * <p> * This token stream provides access to all tokens by index or when calling * methods like {@link #getText}. The channel filtering is only used for code * accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and * {@link #LB}.</p> * * <p> * By default, tokens are placed on the default channel * ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the * {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to * call {@link Lexer#setChannel}. * </p> * * <p> * Note: lexer rules which use the {@code ->skip} lexer command or call * {@link Lexer#skip} do not produce tokens at all, so input text matched by * such a rule will not be available as part of the token stream, regardless of * channel.</p>we */
public class CommonTokenStream extends BufferedTokenStream {
Specifies the channel to use for filtering tokens.

The default value is Token.DEFAULT_CHANNEL, which matches the default channel assigned to tokens created by the lexer.

/** * Specifies the channel to use for filtering tokens. * * <p> * The default value is {@link Token#DEFAULT_CHANNEL}, which matches the * default channel assigned to tokens created by the lexer.</p> */
protected int channel = Token.DEFAULT_CHANNEL;
Constructs a new CommonTokenStream using the specified token source and the default token channel (Token.DEFAULT_CHANNEL).
Params:
  • tokenSource – The token source.
/** * Constructs a new {@link CommonTokenStream} using the specified token * source and the default token channel ({@link Token#DEFAULT_CHANNEL}). * * @param tokenSource The token source. */
public CommonTokenStream(TokenSource tokenSource) { super(tokenSource); }
Constructs a new CommonTokenStream using the specified token source and filtering tokens to the specified channel. Only tokens whose Token.getChannel matches channel or have the Token.getType equal to Token.EOF will be returned by the token stream lookahead methods.
Params:
  • tokenSource – The token source.
  • channel – The channel to use for filtering tokens.
/** * Constructs a new {@link CommonTokenStream} using the specified token * source and filtering tokens to the specified channel. Only tokens whose * {@link Token#getChannel} matches {@code channel} or have the * {@link Token#getType} equal to {@link Token#EOF} will be returned by the * token stream lookahead methods. * * @param tokenSource The token source. * @param channel The channel to use for filtering tokens. */
public CommonTokenStream(TokenSource tokenSource, int channel) { this(tokenSource); this.channel = channel; } @Override protected int adjustSeekIndex(int i) { return nextTokenOnChannel(i, channel); } @Override protected Token LB(int k) { if ( k==0 || (p-k)<0 ) return null; int i = p; int n = 1; // find k good tokens looking backwards while ( n<=k && i>0 ) { // skip off-channel tokens i = previousTokenOnChannel(i - 1, channel); n++; } if ( i<0 ) return null; return tokens.get(i); } @Override public Token LT(int k) { //System.out.println("enter LT("+k+")"); lazyInit(); if ( k == 0 ) return null; if ( k < 0 ) return LB(-k); int i = p; int n = 1; // we know tokens[p] is a good one // find k good tokens while ( n<k ) { // skip off-channel tokens, but make sure to not look past EOF if (sync(i + 1)) { i = nextTokenOnChannel(i + 1, channel); } n++; } // if ( i>range ) range = i; return tokens.get(i); }
Count EOF just once.
/** Count EOF just once. */
public int getNumberOfOnChannelTokens() { int n = 0; fill(); for (int i = 0; i < tokens.size(); i++) { Token t = tokens.get(i); if ( t.getChannel()==channel ) n++; if ( t.getType()==Token.EOF ) break; } return n; } }