/*
 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
 * Use of this file is governed by the BSD 3-clause license that
 * can be found in the LICENSE.txt file in the project root.
 */

package org.antlr.v4.runtime;

import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.Pair;

This default implementation of TokenFactory creates CommonToken objects.
/** * This default implementation of {@link TokenFactory} creates * {@link CommonToken} objects. */
public class CommonTokenFactory implements TokenFactory<CommonToken> {
The default CommonTokenFactory instance.

This token factory does not explicitly copy token text when constructing tokens.

/** * The default {@link CommonTokenFactory} instance. * * <p> * This token factory does not explicitly copy token text when constructing * tokens.</p> */
public static final TokenFactory<CommonToken> DEFAULT = new CommonTokenFactory();
Indicates whether CommonToken.setText should be called after constructing tokens to explicitly set the text. This is useful for cases where the input stream might not be able to provide arbitrary substrings of text from the input after the lexer creates a token (e.g. the implementation of CharStream.getText in UnbufferedCharStream throws an UnsupportedOperationException). Explicitly setting the token text allows Token.getText to be called at any time regardless of the input stream implementation.

The default value is false to avoid the performance and memory overhead of copying text for every token unless explicitly requested.

/** * Indicates whether {@link CommonToken#setText} should be called after * constructing tokens to explicitly set the text. This is useful for cases * where the input stream might not be able to provide arbitrary substrings * of text from the input after the lexer creates a token (e.g. the * implementation of {@link CharStream#getText} in * {@link UnbufferedCharStream} throws an * {@link UnsupportedOperationException}). Explicitly setting the token text * allows {@link Token#getText} to be called at any time regardless of the * input stream implementation. * * <p> * The default value is {@code false} to avoid the performance and memory * overhead of copying text for every token unless explicitly requested.</p> */
protected final boolean copyText;
Constructs a CommonTokenFactory with the specified value for copyText.

When copyText is false, the DEFAULT instance should be used instead of constructing a new instance.

Params:
/** * Constructs a {@link CommonTokenFactory} with the specified value for * {@link #copyText}. * * <p> * When {@code copyText} is {@code false}, the {@link #DEFAULT} instance * should be used instead of constructing a new instance.</p> * * @param copyText The value for {@link #copyText}. */
public CommonTokenFactory(boolean copyText) { this.copyText = copyText; }
Constructs a CommonTokenFactory with copyText set to false.

The DEFAULT instance should be used instead of calling this directly.

/** * Constructs a {@link CommonTokenFactory} with {@link #copyText} set to * {@code false}. * * <p> * The {@link #DEFAULT} instance should be used instead of calling this * directly.</p> */
public CommonTokenFactory() { this(false); } @Override public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line, int charPositionInLine) { CommonToken t = new CommonToken(source, type, channel, start, stop); t.setLine(line); t.setCharPositionInLine(charPositionInLine); if ( text!=null ) { t.setText(text); } else if ( copyText && source.b != null ) { t.setText(source.b.getText(Interval.of(start,stop))); } return t; } @Override public CommonToken create(int type, String text) { return new CommonToken(type, text); } }