package org.graalvm.compiler.phases.common;
import static org.graalvm.compiler.core.common.GraalOptions.OptEliminateGuards;
import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED;
import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED;
import static org.graalvm.compiler.phases.common.LoweringPhase.ProcessBlockState.ST_ENTER;
import static org.graalvm.compiler.phases.common.LoweringPhase.ProcessBlockState.ST_ENTER_ALWAYS_REACHED;
import static org.graalvm.compiler.phases.common.LoweringPhase.ProcessBlockState.ST_LEAVE;
import static org.graalvm.compiler.phases.common.LoweringPhase.ProcessBlockState.ST_PROCESS;
import static org.graalvm.compiler.phases.common.LoweringPhase.ProcessBlockState.ST_PROCESS_ALWAYS_REACHED;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.graalvm.compiler.core.common.type.StampFactory;
import org.graalvm.compiler.debug.DebugCloseable;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.graph.Graph.Mark;
import org.graalvm.compiler.graph.Node;
import org.graalvm.compiler.graph.NodeBitMap;
import org.graalvm.compiler.graph.NodeClass;
import org.graalvm.compiler.graph.NodeMap;
import org.graalvm.compiler.graph.NodeSourcePosition;
import org.graalvm.compiler.graph.iterators.NodeIterable;
import org.graalvm.compiler.nodeinfo.InputType;
import org.graalvm.compiler.nodeinfo.NodeInfo;
import org.graalvm.compiler.nodes.AbstractBeginNode;
import org.graalvm.compiler.nodes.BeginNode;
import org.graalvm.compiler.nodes.ControlSinkNode;
import org.graalvm.compiler.nodes.FixedGuardNode;
import org.graalvm.compiler.nodes.FixedNode;
import org.graalvm.compiler.nodes.FixedWithNextNode;
import org.graalvm.compiler.nodes.GuardNode;
import org.graalvm.compiler.nodes.LogicNode;
import org.graalvm.compiler.nodes.PhiNode;
import org.graalvm.compiler.nodes.ProxyNode;
import org.graalvm.compiler.nodes.StructuredGraph;
import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult;
import org.graalvm.compiler.nodes.ValueNode;
import org.graalvm.compiler.nodes.calc.FloatingNode;
import org.graalvm.compiler.nodes.cfg.Block;
import org.graalvm.compiler.nodes.extended.AnchoringNode;
import org.graalvm.compiler.nodes.extended.GuardedNode;
import org.graalvm.compiler.nodes.extended.GuardingNode;
import org.graalvm.compiler.nodes.memory.MemoryKill;
import org.graalvm.compiler.nodes.memory.MultiMemoryKill;
import org.graalvm.compiler.nodes.memory.SingleMemoryKill;
import org.graalvm.compiler.nodes.spi.CoreProviders;
import org.graalvm.compiler.nodes.spi.CoreProvidersDelegate;
import org.graalvm.compiler.nodes.spi.Lowerable;
import org.graalvm.compiler.nodes.spi.LoweringTool;
import org.graalvm.compiler.options.OptionValues;
import org.graalvm.compiler.phases.BasePhase;
import org.graalvm.compiler.phases.Phase;
import org.graalvm.compiler.phases.schedule.SchedulePhase;
import jdk.internal.vm.compiler.word.LocationIdentity;
import jdk.vm.ci.meta.DeoptimizationAction;
import jdk.vm.ci.meta.DeoptimizationReason;
import jdk.vm.ci.meta.SpeculationLog;
import jdk.vm.ci.meta.SpeculationLog.Speculation;
public class LoweringPhase extends BasePhase<CoreProviders> {
@NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED)
static final class DummyGuardHandle extends ValueNode implements GuardedNode {
public static final NodeClass<DummyGuardHandle> TYPE = NodeClass.create(DummyGuardHandle.class);
@Input(InputType.Guard) GuardingNode guard;
protected DummyGuardHandle(GuardingNode guard) {
super(TYPE, StampFactory.forVoid());
this.guard = guard;
}
@Override
public GuardingNode getGuard() {
return guard;
}
@Override
public void setGuard(GuardingNode guard) {
updateUsagesInterface(this.guard, guard);
this.guard = guard;
}
@Override
public ValueNode asNode() {
return this;
}
}
@Override
public boolean checkContract() {
return false;
}
final class LoweringToolImpl extends CoreProvidersDelegate implements LoweringTool {
private final NodeBitMap activeGuards;
private AnchoringNode guardAnchor;
private FixedWithNextNode lastFixedNode;
private NodeMap<Block> nodeMap;
LoweringToolImpl(CoreProviders context, AnchoringNode guardAnchor, NodeBitMap activeGuards, FixedWithNextNode lastFixedNode, NodeMap<Block> nodeMap) {
super(context);
this.guardAnchor = guardAnchor;
this.activeGuards = activeGuards;
this.lastFixedNode = lastFixedNode;
this.nodeMap = nodeMap;
}
@Override
public LoweringStage getLoweringStage() {
return loweringStage;
}
@Override
public AnchoringNode getCurrentGuardAnchor() {
return guardAnchor;
}
@Override
public GuardingNode createGuard(FixedNode before, LogicNode condition, DeoptimizationReason deoptReason, DeoptimizationAction action) {
return createGuard(before, condition, deoptReason, action, SpeculationLog.NO_SPECULATION, false, null);
}
@Override
public GuardingNode createGuard(FixedNode before, LogicNode condition, DeoptimizationReason deoptReason, DeoptimizationAction action, Speculation speculation, boolean negated,
NodeSourcePosition noDeoptSucccessorPosition) {
StructuredGraph graph = before.graph();
if (OptEliminateGuards.getValue(graph.getOptions())) {
for (Node usage : condition.usages()) {
if (!activeGuards.isNew(usage) && activeGuards.isMarked(usage) && ((GuardNode) usage).isNegated() == negated &&
(!before.graph().hasValueProxies() || nodeMap.get(((GuardNode) usage).getAnchor().asNode()).isInSameOrOuterLoopOf(nodeMap.get(before)))) {
return (GuardNode) usage;
}
}
}
if (!condition.graph().getGuardsStage().allowsFloatingGuards()) {
FixedGuardNode fixedGuard = graph.add(new FixedGuardNode(condition, deoptReason, action, speculation, negated, noDeoptSucccessorPosition));
graph.addBeforeFixed(before, fixedGuard);
DummyGuardHandle handle = graph.add(new DummyGuardHandle(fixedGuard));
fixedGuard.lower(this);
GuardingNode result = handle.getGuard();
handle.safeDelete();
return result;
} else {
GuardNode newGuard = graph.unique(new GuardNode(condition, guardAnchor, deoptReason, action, negated, speculation, noDeoptSucccessorPosition));
if (OptEliminateGuards.getValue(graph.getOptions())) {
activeGuards.markAndGrow(newGuard);
}
return newGuard;
}
}
@Override
public FixedWithNextNode lastFixedNode() {
return lastFixedNode;
}
private void setLastFixedNode(FixedWithNextNode n) {
assert n.isAlive() : n;
lastFixedNode = n;
}
}
private final CanonicalizerPhase canonicalizer;
private final LoweringTool.LoweringStage loweringStage;
public LoweringPhase(CanonicalizerPhase canonicalizer, LoweringTool.LoweringStage loweringStage) {
this.canonicalizer = canonicalizer;
this.loweringStage = loweringStage;
}
@Override
protected boolean shouldDumpBeforeAtBasicLevel() {
return loweringStage == LoweringTool.StandardLoweringStage.HIGH_TIER;
}
private boolean checkPostLowering(StructuredGraph graph, CoreProviders context) {
Mark expectedMark = graph.getMark();
lower(graph, context, LoweringMode.VERIFY_LOWERING);
Mark mark = graph.getMark();
assert mark.equals(expectedMark) : graph + ": a second round in the current lowering phase introduced these new nodes: " + graph.getNewNodes(expectedMark).snapshot();
return true;
}
@Override
protected void run(final StructuredGraph graph, CoreProviders context) {
lower(graph, context, LoweringMode.LOWERING);
assert checkPostLowering(graph, context);
}
private void lower(StructuredGraph graph, CoreProviders context, LoweringMode mode) {
IncrementalCanonicalizerPhase<CoreProviders> incrementalCanonicalizer = new IncrementalCanonicalizerPhase<>(canonicalizer);
incrementalCanonicalizer.appendPhase(new Round(context, mode, graph.getOptions()));
incrementalCanonicalizer.apply(graph, context);
assert graph.verify();
}
private static boolean checkPostNodeLowering(Node node, LoweringToolImpl loweringTool, Mark preLoweringMark, Collection<Node> unscheduledUsages) {
StructuredGraph graph = (StructuredGraph) node.graph();
Mark postLoweringMark = graph.getMark();
NodeIterable<Node> newNodesAfterLowering = graph.getNewNodes(preLoweringMark);
if (node instanceof FloatingNode) {
if (!unscheduledUsages.isEmpty()) {
for (Node n : newNodesAfterLowering) {
assert !(n instanceof FixedNode) : node.graph() + ": cannot lower floatable node " + node + " as it introduces fixed node(s) but has the following unscheduled usages: " +
unscheduledUsages;
}
}
}
for (Node n : newNodesAfterLowering) {
if (n instanceof Lowerable) {
((Lowerable) n).lower(loweringTool);
Mark mark = graph.getMark();
assert postLoweringMark.equals(mark) : graph + ": lowering of " + node + " produced lowerable " + n + " that should have been recursively lowered as it introduces these new nodes: " +
graph.getNewNodes(postLoweringMark).snapshot();
}
if (graph.isAfterFloatingReadPhase() && n instanceof MemoryKill && !(node instanceof MemoryKill) && !(node instanceof ControlSinkNode)) {
boolean isAny = false;
if (n instanceof SingleMemoryKill) {
isAny = ((SingleMemoryKill) n).getKilledLocationIdentity().isAny();
} else if (n instanceof MultiMemoryKill) {
for (LocationIdentity ident : ((MultiMemoryKill) n).getKilledLocationIdentities()) {
if (ident.isAny()) {
isAny = true;
}
}
} else {
throw GraalError.shouldNotReachHere("Unknown type of memory kill " + n);
}
if (isAny && n instanceof FixedWithNextNode) {
FixedWithNextNode cur = (FixedWithNextNode) n;
while (cur != null && graph.isNew(preLoweringMark, cur)) {
if (cur.next() instanceof ControlSinkNode) {
isAny = false;
break;
}
if (cur.next() instanceof FixedWithNextNode) {
cur = (FixedWithNextNode) cur.next();
} else {
break;
}
}
}
assert !isAny : node + " " + n;
}
}
return true;
}
private enum LoweringMode {
LOWERING,
VERIFY_LOWERING
}
private final class Round extends Phase {
private final CoreProviders context;
private final LoweringMode mode;
private ScheduleResult schedule;
private final SchedulePhase schedulePhase;
private Round(CoreProviders context, LoweringMode mode, OptionValues options) {
this.context = context;
this.mode = mode;
boolean immutableSchedule = mode == LoweringMode.VERIFY_LOWERING;
this.schedulePhase = new SchedulePhase(immutableSchedule, options);
}
@Override
protected CharSequence getName() {
switch (mode) {
case LOWERING:
return "LoweringRound";
case VERIFY_LOWERING:
return "VerifyLoweringRound";
default:
throw GraalError.shouldNotReachHere();
}
}
@Override
public boolean checkContract() {
return false;
}
@Override
public void run(StructuredGraph graph) {
schedulePhase.apply(graph, false);
schedule = graph.getLastSchedule();
schedule.getCFG().computePostdominators();
Block startBlock = schedule.getCFG().getStartBlock();
ProcessFrame rootFrame = new ProcessFrame(startBlock, graph.createNodeBitMap(), startBlock.getBeginNode(), null);
LoweringPhase.processBlock(rootFrame);
}
private class ProcessFrame extends Frame<ProcessFrame> {
private final NodeBitMap activeGuards;
private AnchoringNode anchor;
ProcessFrame(Block block, NodeBitMap activeGuards, AnchoringNode anchor, ProcessFrame parent) {
super(block, parent);
this.activeGuards = activeGuards;
this.anchor = anchor;
}
@Override
public void preprocess() {
this.anchor = Round.this.process(block, activeGuards, anchor);
}
@Override
public ProcessFrame enter(Block b) {
return new ProcessFrame(b, activeGuards, b.getBeginNode(), this);
}
@Override
public Frame<?> enterAlwaysReached(Block b) {
AnchoringNode newAnchor = anchor;
if (parent != null && b.getLoop() != parent.block.getLoop() && !b.isLoopHeader()) {
newAnchor = b.getBeginNode();
}
return new ProcessFrame(b, activeGuards, newAnchor, this);
}
@Override
public void postprocess() {
if (anchor == block.getBeginNode() && OptEliminateGuards.getValue(activeGuards.graph().getOptions())) {
for (GuardNode guard : anchor.asNode().usages().filter(GuardNode.class)) {
if (activeGuards.isMarkedAndGrow(guard)) {
activeGuards.clear(guard);
}
}
}
}
}
@SuppressWarnings("try")
private AnchoringNode process(final Block b, final NodeBitMap activeGuards, final AnchoringNode startAnchor) {
final LoweringToolImpl loweringTool = new LoweringToolImpl(context, startAnchor, activeGuards, b.getBeginNode(), this.schedule.getNodeToBlockMap());
List<Node> nodes = schedule.nodesFor(b);
for (Node node : nodes) {
if (node.isDeleted()) {
continue;
}
FixedNode nextNode = null;
if (node instanceof FixedWithNextNode) {
nextNode = ((FixedWithNextNode) node).next();
} else {
nextNode = loweringTool.lastFixedNode().next();
}
if (node instanceof Lowerable) {
Collection<Node> unscheduledUsages = null;
assert (unscheduledUsages = getUnscheduledUsages(node)) != null;
Mark preLoweringMark = node.graph().getMark();
try (DebugCloseable s = node.graph().withNodeSourcePosition(node)) {
((Lowerable) node).lower(loweringTool);
}
if (loweringTool.guardAnchor.asNode().isDeleted()) {
assert nextNode.isAlive();
loweringTool.guardAnchor = AbstractBeginNode.prevBegin(nextNode);
}
assert checkPostNodeLowering(node, loweringTool, preLoweringMark, unscheduledUsages);
}
if (!nextNode.isAlive()) {
break;
} else {
Node nextLastFixed = nextNode.predecessor();
if (!(nextLastFixed instanceof FixedWithNextNode)) {
AbstractBeginNode begin = node.graph().add(new BeginNode());
nextLastFixed.replaceFirstSuccessor(nextNode, begin);
begin.setNext(nextNode);
nextLastFixed = begin;
}
loweringTool.setLastFixedNode((FixedWithNextNode) nextLastFixed);
}
}
return loweringTool.getCurrentGuardAnchor();
}
private Collection<Node> getUnscheduledUsages(Node node) {
List<Node> unscheduledUsages = new ArrayList<>();
if (node instanceof FloatingNode) {
for (Node usage : node.usages()) {
if (usage instanceof ValueNode && !(usage instanceof PhiNode) && !(usage instanceof ProxyNode)) {
if (schedule.getCFG().getNodeToBlock().isNew(usage) || schedule.getCFG().blockFor(usage) == null) {
unscheduledUsages.add(usage);
}
}
}
}
return unscheduledUsages;
}
}
enum ProcessBlockState {
ST_ENTER,
ST_PROCESS,
ST_ENTER_ALWAYS_REACHED,
ST_LEAVE,
ST_PROCESS_ALWAYS_REACHED;
}
public static void processBlock(final Frame<?> rootFrame) {
ProcessBlockState state = ST_PROCESS;
Frame<?> f = rootFrame;
while (f != null) {
ProcessBlockState nextState;
if (state == ST_PROCESS || state == ST_PROCESS_ALWAYS_REACHED) {
f.preprocess();
nextState = state == ST_PROCESS_ALWAYS_REACHED ? ST_ENTER : ST_ENTER_ALWAYS_REACHED;
} else if (state == ST_ENTER_ALWAYS_REACHED) {
if (f.alwaysReachedBlock != null && f.alwaysReachedBlock.getDominator() == f.block) {
f = f.enterAlwaysReached(f.alwaysReachedBlock);
nextState = ST_PROCESS;
} else {
nextState = ST_ENTER;
}
} else if (state == ST_ENTER) {
if (f.dominated != null) {
Block n = f.dominated;
f.dominated = n.getDominatedSibling();
if (n == f.alwaysReachedBlock) {
if (f.dominated != null) {
n = f.dominated;
f.dominated = n.getDominatedSibling();
} else {
n = null;
}
}
if (n == null) {
nextState = ST_LEAVE;
} else {
f = f.enter(n);
assert f.block.getDominator() == f.parent.block;
nextState = ST_PROCESS;
}
} else {
nextState = ST_LEAVE;
}
} else if (state == ST_LEAVE) {
f.postprocess();
f = f.parent;
nextState = ST_ENTER;
} else {
throw GraalError.shouldNotReachHere();
}
state = nextState;
}
}
public abstract static class Frame<T extends Frame<?>> {
protected final Block block;
final T parent;
Block dominated;
final Block alwaysReachedBlock;
public Frame(Block block, T parent) {
this.block = block;
this.alwaysReachedBlock = block.getPostdominator();
this.dominated = block.getFirstDominated();
this.parent = parent;
}
public Frame<?> enterAlwaysReached(Block b) {
return enter(b);
}
public abstract Frame<?> enter(Block b);
public abstract void preprocess();
public abstract void postprocess();
}
}