/*
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.Pair;
This default implementation of TokenFactory
creates CommonToken
objects. /**
* This default implementation of {@link TokenFactory} creates
* {@link CommonToken} objects.
*/
public class CommonTokenFactory implements TokenFactory<CommonToken> {
The default CommonTokenFactory
instance.
This token factory does not explicitly copy token text when constructing
tokens.
/**
* The default {@link CommonTokenFactory} instance.
*
* <p>
* This token factory does not explicitly copy token text when constructing
* tokens.</p>
*/
public static final TokenFactory<CommonToken> DEFAULT = new CommonTokenFactory();
Indicates whether CommonToken.setText
should be called after constructing tokens to explicitly set the text. This is useful for cases where the input stream might not be able to provide arbitrary substrings of text from the input after the lexer creates a token (e.g. the implementation of CharStream.getText
in UnbufferedCharStream
throws an UnsupportedOperationException
). Explicitly setting the token text allows Token.getText
to be called at any time regardless of the input stream implementation. The default value is false
to avoid the performance and memory overhead of copying text for every token unless explicitly requested.
/**
* Indicates whether {@link CommonToken#setText} should be called after
* constructing tokens to explicitly set the text. This is useful for cases
* where the input stream might not be able to provide arbitrary substrings
* of text from the input after the lexer creates a token (e.g. the
* implementation of {@link CharStream#getText} in
* {@link UnbufferedCharStream} throws an
* {@link UnsupportedOperationException}). Explicitly setting the token text
* allows {@link Token#getText} to be called at any time regardless of the
* input stream implementation.
*
* <p>
* The default value is {@code false} to avoid the performance and memory
* overhead of copying text for every token unless explicitly requested.</p>
*/
protected final boolean copyText;
Constructs a CommonTokenFactory
with the specified value for copyText
. When copyText
is false
, the DEFAULT
instance should be used instead of constructing a new instance.
Params: - copyText – The value for
copyText
.
/**
* Constructs a {@link CommonTokenFactory} with the specified value for
* {@link #copyText}.
*
* <p>
* When {@code copyText} is {@code false}, the {@link #DEFAULT} instance
* should be used instead of constructing a new instance.</p>
*
* @param copyText The value for {@link #copyText}.
*/
public CommonTokenFactory(boolean copyText) { this.copyText = copyText; }
Constructs a CommonTokenFactory
with copyText
set to false
. The DEFAULT
instance should be used instead of calling this directly.
/**
* Constructs a {@link CommonTokenFactory} with {@link #copyText} set to
* {@code false}.
*
* <p>
* The {@link #DEFAULT} instance should be used instead of calling this
* directly.</p>
*/
public CommonTokenFactory() { this(false); }
@Override
public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text,
int channel, int start, int stop,
int line, int charPositionInLine)
{
CommonToken t = new CommonToken(source, type, channel, start, stop);
t.setLine(line);
t.setCharPositionInLine(charPositionInLine);
if ( text!=null ) {
t.setText(text);
}
else if ( copyText && source.b != null ) {
t.setText(source.b.getText(Interval.of(start,stop)));
}
return t;
}
@Override
public CommonToken create(int type, String text) {
return new CommonToken(type, text);
}
}