package org.graalvm.compiler.replacements;
import static org.graalvm.compiler.core.common.GraalOptions.UseSnippetGraphCache;
import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM;
import static org.graalvm.compiler.java.BytecodeParserOptions.InlineDuringParsing;
import static org.graalvm.compiler.java.BytecodeParserOptions.InlineIntrinsicsDuringParsing;
import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createIntrinsicInlineInfo;
import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING;
import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import jdk.internal.vm.compiler.collections.EconomicMap;
import jdk.internal.vm.compiler.collections.Equivalence;
import org.graalvm.compiler.api.replacements.Fold;
import org.graalvm.compiler.api.replacements.MethodSubstitution;
import org.graalvm.compiler.api.replacements.Snippet;
import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
import org.graalvm.compiler.api.replacements.SnippetTemplateCache;
import org.graalvm.compiler.bytecode.Bytecode;
import org.graalvm.compiler.bytecode.BytecodeProvider;
import org.graalvm.compiler.bytecode.ResolvedJavaMethodBytecode;
import org.graalvm.compiler.core.common.GraalOptions;
import org.graalvm.compiler.core.common.spi.ConstantFieldProvider;
import org.graalvm.compiler.debug.DebugCloseable;
import org.graalvm.compiler.debug.DebugContext;
import org.graalvm.compiler.debug.DebugContext.Description;
import org.graalvm.compiler.debug.DebugHandlersFactory;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.debug.TimerKey;
import org.graalvm.compiler.graph.Node;
import org.graalvm.compiler.graph.NodeSourcePosition;
import org.graalvm.compiler.graph.Node.NodeIntrinsic;
import org.graalvm.compiler.java.GraphBuilderPhase;
import org.graalvm.compiler.java.GraphBuilderPhase.Instance;
import org.graalvm.compiler.nodes.CallTargetNode;
import org.graalvm.compiler.nodes.Invoke;
import org.graalvm.compiler.nodes.StateSplit;
import org.graalvm.compiler.nodes.StructuredGraph;
import org.graalvm.compiler.nodes.ValueNode;
import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedInvocationPlugin;
import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration;
import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin;
import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext;
import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin;
import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
import org.graalvm.compiler.nodes.spi.Replacements;
import org.graalvm.compiler.nodes.spi.StampProvider;
import org.graalvm.compiler.options.OptionValues;
import org.graalvm.compiler.phases.OptimisticOptimizations;
import org.graalvm.compiler.phases.common.CanonicalizerPhase;
import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase;
import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase;
import org.graalvm.compiler.phases.tiers.PhaseContext;
import org.graalvm.compiler.phases.util.Providers;
import org.graalvm.compiler.word.Word;
import org.graalvm.compiler.word.WordOperationPlugin;
import jdk.vm.ci.code.TargetDescription;
import jdk.vm.ci.meta.ConstantReflectionProvider;
import jdk.vm.ci.meta.MetaAccessProvider;
import jdk.vm.ci.meta.ResolvedJavaMethod;
import jdk.vm.ci.meta.ResolvedJavaType;
public class ReplacementsImpl implements Replacements, InlineInvokePlugin {
protected final OptionValues options;
public final Providers providers;
public final SnippetReflectionProvider snippetReflection;
public final TargetDescription target;
private GraphBuilderConfiguration.Plugins graphBuilderPlugins;
private final DebugHandlersFactory debugHandlersFactory;
@Override
public OptionValues getOptions() {
return options;
}
protected final ConcurrentMap<ResolvedJavaMethod, StructuredGraph> graphs;
protected final BytecodeProvider defaultBytecodeProvider;
public void setGraphBuilderPlugins(GraphBuilderConfiguration.Plugins plugins) {
assert this.graphBuilderPlugins == null;
this.graphBuilderPlugins = plugins;
}
@Override
public GraphBuilderConfiguration.Plugins getGraphBuilderPlugins() {
return graphBuilderPlugins;
}
protected boolean hasGeneratedInvocationPluginAnnotation(ResolvedJavaMethod method) {
return method.getAnnotation(Node.NodeIntrinsic.class) != null || method.getAnnotation(Fold.class) != null;
}
protected boolean hasGenericInvocationPluginAnnotation(ResolvedJavaMethod method) {
return method.getAnnotation(Word.Operation.class) != null;
}
private static final int MAX_GRAPH_INLINING_DEPTH = 100;
@Override
public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) {
Bytecode subst = getSubstitutionBytecode(method);
if (subst != null) {
if (b.parsingIntrinsic() || InlineDuringParsing.getValue(b.getOptions()) || InlineIntrinsicsDuringParsing.getValue(b.getOptions())) {
return createIntrinsicInlineInfo(subst.getMethod(), method, subst.getOrigin());
}
return null;
}
if (b.parsingIntrinsic()) {
assert b.getDepth() < MAX_GRAPH_INLINING_DEPTH : "inlining limit exceeded";
return createIntrinsicInlineInfo(method, null, defaultBytecodeProvider);
} else {
assert method.getAnnotation(NodeIntrinsic.class) == null : String.format("@%s method %s must only be called from within a replacement%n%s", NodeIntrinsic.class.getSimpleName(),
method.format("%h.%n"), b);
}
return null;
}
@Override
public void notifyNotInlined(GraphBuilderContext b, ResolvedJavaMethod method, Invoke invoke) {
if (b.parsingIntrinsic()) {
IntrinsicContext intrinsic = b.getIntrinsic();
if (!intrinsic.isCallToOriginal(method)) {
if (hasGeneratedInvocationPluginAnnotation(method)) {
throw new GraalError("%s should have been handled by a %s", method.format("%H.%n(%p)"), GeneratedInvocationPlugin.class.getSimpleName());
}
if (hasGenericInvocationPluginAnnotation(method)) {
throw new GraalError("%s should have been handled by %s", method.format("%H.%n(%p)"), WordOperationPlugin.class.getSimpleName());
}
throw new GraalError("All non-recursive calls in the intrinsic %s must be inlined or intrinsified: found call to %s",
intrinsic.getIntrinsicMethod().format("%H.%n(%p)"), method.format("%h.%n(%p)"));
}
}
}
private final EconomicMap<String, SnippetTemplateCache> snippetTemplateCache;
public ReplacementsImpl(OptionValues options, DebugHandlersFactory debugHandlersFactory, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider,
TargetDescription target) {
this.options = options;
this.providers = providers.copyWith(this);
this.snippetReflection = snippetReflection;
this.target = target;
this.graphs = new ConcurrentHashMap<>();
this.snippetTemplateCache = EconomicMap.create(Equivalence.DEFAULT);
this.defaultBytecodeProvider = bytecodeProvider;
this.debugHandlersFactory = debugHandlersFactory;
}
private static final TimerKey SnippetPreparationTime = DebugContext.timer("SnippetPreparationTime");
@Override
public StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) {
return getSnippet(method, null, args, trackNodeSourcePosition, replaceePosition);
}
private static final AtomicInteger nextDebugContextId = new AtomicInteger();
protected DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) {
DebugContext outer = DebugContext.forCurrentThread();
Description description = new Description(method, idPrefix + nextDebugContextId.incrementAndGet());
List<DebugHandlersFactory> factories = debugHandlersFactory == null ? Collections.emptyList() : Collections.singletonList(debugHandlersFactory);
return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories);
}
@Override
@SuppressWarnings("try")
public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) {
assert method.getAnnotation(Snippet.class) != null : "Snippet must be annotated with @" + Snippet.class.getSimpleName();
assert method.hasBytecodes() : "Snippet must not be abstract or native";
StructuredGraph graph = UseSnippetGraphCache.getValue(options) ? graphs.get(method) : null;
if (graph == null || (trackNodeSourcePosition && !graph.trackNodeSourcePosition())) {
try (DebugContext debug = openDebugContext("Snippet_", method);
DebugCloseable a = SnippetPreparationTime.start(debug)) {
StructuredGraph newGraph = makeGraph(debug, defaultBytecodeProvider, method, args, recursiveEntry, trackNodeSourcePosition, replaceePosition);
DebugContext.counter("SnippetNodeCount[%#s]", method).add(newGraph.getDebug(), newGraph.getNodeCount());
if (!UseSnippetGraphCache.getValue(options) || args != null) {
return newGraph;
}
newGraph.freeze();
if (graph != null) {
graphs.replace(method, graph, newGraph);
} else {
graphs.putIfAbsent(method, newGraph);
}
graph = graphs.get(method);
}
}
assert !trackNodeSourcePosition || graph.trackNodeSourcePosition();
return graph;
}
@Override
public void registerSnippet(ResolvedJavaMethod method, boolean trackNodeSourcePosition) {
}
@Override
public boolean hasSubstitution(ResolvedJavaMethod method, int invokeBci) {
InvocationPlugin plugin = graphBuilderPlugins.getInvocationPlugins().lookupInvocation(method);
return plugin != null && (!plugin.inlineOnly() || invokeBci >= 0);
}
@Override
public BytecodeProvider getDefaultReplacementBytecodeProvider() {
return defaultBytecodeProvider;
}
@Override
public Bytecode getSubstitutionBytecode(ResolvedJavaMethod method) {
InvocationPlugin plugin = graphBuilderPlugins.getInvocationPlugins().lookupInvocation(method);
if (plugin instanceof MethodSubstitutionPlugin) {
MethodSubstitutionPlugin msPlugin = (MethodSubstitutionPlugin) plugin;
ResolvedJavaMethod substitute = msPlugin.getSubstitute(providers.getMetaAccess());
return msPlugin.getBytecodeProvider().getBytecode(substitute);
}
return null;
}
@Override
public StructuredGraph getSubstitution(ResolvedJavaMethod method, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) {
StructuredGraph result;
InvocationPlugin plugin = graphBuilderPlugins.getInvocationPlugins().lookupInvocation(method);
if (plugin != null && (!plugin.inlineOnly() || invokeBci >= 0)) {
MetaAccessProvider metaAccess = providers.getMetaAccess();
if (plugin instanceof MethodSubstitutionPlugin) {
MethodSubstitutionPlugin msPlugin = (MethodSubstitutionPlugin) plugin;
ResolvedJavaMethod substitute = msPlugin.getSubstitute(metaAccess);
StructuredGraph graph = UseSnippetGraphCache.getValue(options) ? graphs.get(substitute) : null;
if (graph == null || graph.trackNodeSourcePosition() != trackNodeSourcePosition) {
try (DebugContext debug = openDebugContext("Substitution_", method)) {
graph = makeGraph(debug, msPlugin.getBytecodeProvider(), substitute, null, method, trackNodeSourcePosition, replaceePosition);
if (!UseSnippetGraphCache.getValue(options)) {
return graph;
}
graph.freeze();
graphs.putIfAbsent(substitute, graph);
graph = graphs.get(substitute);
}
}
assert graph.isFrozen();
result = graph;
} else {
Bytecode code = new ResolvedJavaMethodBytecode(method);
ConstantReflectionProvider constantReflection = providers.getConstantReflection();
ConstantFieldProvider constantFieldProvider = providers.getConstantFieldProvider();
StampProvider stampProvider = providers.getStampProvider();
try (DebugContext debug = openDebugContext("Substitution_", method)) {
result = new IntrinsicGraphBuilder(options, debug, metaAccess, constantReflection, constantFieldProvider, stampProvider, code, invokeBci).buildGraph(plugin);
}
}
} else {
result = null;
}
return result;
}
public StructuredGraph makeGraph(DebugContext debug, BytecodeProvider bytecodeProvider, ResolvedJavaMethod method, Object[] args, ResolvedJavaMethod original, boolean trackNodeSourcePosition,
NodeSourcePosition replaceePosition) {
return createGraphMaker(method, original).makeGraph(debug, bytecodeProvider, args, trackNodeSourcePosition, replaceePosition);
}
protected GraphMaker createGraphMaker(ResolvedJavaMethod substitute, ResolvedJavaMethod original) {
return new GraphMaker(this, substitute, original);
}
public static class GraphMaker {
protected final ReplacementsImpl replacements;
protected final ResolvedJavaMethod method;
protected final ResolvedJavaMethod substitutedMethod;
protected GraphMaker(ReplacementsImpl replacements, ResolvedJavaMethod substitute, ResolvedJavaMethod substitutedMethod) {
this.replacements = replacements;
this.method = substitute;
this.substitutedMethod = substitutedMethod;
}
@SuppressWarnings("try")
public StructuredGraph makeGraph(DebugContext debug, BytecodeProvider bytecodeProvider, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) {
try (DebugContext.Scope s = debug.scope("BuildSnippetGraph", method)) {
assert method.hasBytecodes() : method;
StructuredGraph graph = buildInitialGraph(debug, bytecodeProvider, method, args, trackNodeSourcePosition, replaceePosition);
finalizeGraph(graph);
debug.dump(DebugContext.INFO_LEVEL, graph, "%s: Final", method.getName());
return graph;
} catch (Throwable e) {
throw debug.handle(e);
}
}
protected void finalizeGraph(StructuredGraph graph) {
if (!GraalOptions.SnippetCounters.getValue(replacements.options) || graph.getNodes().filter(SnippetCounterNode.class).isEmpty()) {
int sideEffectCount = 0;
assert (sideEffectCount = graph.getNodes().filter(e -> hasSideEffect(e)).count()) >= 0;
new ConvertDeoptimizeToGuardPhase().apply(graph, null);
assert sideEffectCount == graph.getNodes().filter(e -> hasSideEffect(e)).count() : "deleted side effecting node";
new DeadCodeEliminationPhase(Required).apply(graph);
} else {
}
}
private boolean hasSideEffect(Node node) {
if (node instanceof StateSplit) {
if (((StateSplit) node).hasSideEffect()) {
if (node instanceof Invoke) {
CallTargetNode callTarget = ((Invoke) node).callTarget();
if (callTarget instanceof MethodCallTargetNode) {
ResolvedJavaMethod targetMethod = ((MethodCallTargetNode) callTarget).targetMethod();
if (targetMethod.isConstructor()) {
ResolvedJavaType throwableType = replacements.providers.getMetaAccess().lookupJavaType(Throwable.class);
return !throwableType.isAssignableFrom(targetMethod.getDeclaringClass());
}
}
}
return true;
}
}
return false;
}
@SuppressWarnings("try")
protected StructuredGraph buildInitialGraph(DebugContext debug, BytecodeProvider bytecodeProvider, final ResolvedJavaMethod methodToParse, Object[] args, boolean trackNodeSourcePosition,
NodeSourcePosition replaceePosition) {
final StructuredGraph graph = new StructuredGraph.Builder(replacements.options, debug).method(methodToParse).trackNodeSourcePosition(trackNodeSourcePosition).callerContext(
replaceePosition).build();
graph.disableUnsafeAccessTracking();
try (DebugContext.Scope s = debug.scope("buildInitialGraph", graph)) {
MetaAccessProvider metaAccess = replacements.providers.getMetaAccess();
Plugins plugins = new Plugins(replacements.graphBuilderPlugins);
GraphBuilderConfiguration config = GraphBuilderConfiguration.getSnippetDefault(plugins);
if (args != null) {
plugins.prependParameterPlugin(new ConstantBindingParameterPlugin(args, metaAccess, replacements.snippetReflection));
}
IntrinsicContext initialIntrinsicContext = null;
Snippet snippetAnnotation = method.getAnnotation(Snippet.class);
if (snippetAnnotation == null) {
initialIntrinsicContext = new IntrinsicContext(substitutedMethod, method, bytecodeProvider, INLINE_AFTER_PARSING);
} else {
ResolvedJavaMethod original = substitutedMethod != null ? substitutedMethod : method;
initialIntrinsicContext = new IntrinsicContext(original, method, bytecodeProvider, INLINE_AFTER_PARSING, snippetAnnotation.allowPartialIntrinsicArgumentMismatch());
}
createGraphBuilder(metaAccess, replacements.providers.getStampProvider(), replacements.providers.getConstantReflection(), replacements.providers.getConstantFieldProvider(), config,
OptimisticOptimizations.NONE, initialIntrinsicContext).apply(graph);
new CanonicalizerPhase().apply(graph, new PhaseContext(replacements.providers));
} catch (Throwable e) {
throw debug.handle(e);
}
return graph;
}
protected Instance createGraphBuilder(MetaAccessProvider metaAccess, StampProvider stampProvider, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantFieldProvider,
GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) {
return new GraphBuilderPhase.Instance(metaAccess, stampProvider, constantReflection, constantFieldProvider, graphBuilderConfig, optimisticOpts,
initialIntrinsicContext);
}
}
@Override
public void registerSnippetTemplateCache(SnippetTemplateCache templates) {
assert snippetTemplateCache.get(templates.getClass().getName()) == null;
snippetTemplateCache.put(templates.getClass().getName(), templates);
}
@Override
public <T extends SnippetTemplateCache> T getSnippetTemplateCache(Class<T> templatesClass) {
SnippetTemplateCache ret = snippetTemplateCache.get(templatesClass.getName());
return templatesClass.cast(ret);
}
}