package org.graalvm.compiler.lir.alloc.trace;
import static jdk.vm.ci.code.ValueUtil.asAllocatableValue;
import static jdk.vm.ci.code.ValueUtil.asRegister;
import static jdk.vm.ci.code.ValueUtil.asStackSlot;
import static jdk.vm.ci.code.ValueUtil.isIllegal;
import static jdk.vm.ci.code.ValueUtil.isRegister;
import static jdk.vm.ci.code.ValueUtil.isStackSlot;
import static org.graalvm.compiler.lir.LIRValueUtil.asVirtualStackSlot;
import static org.graalvm.compiler.lir.LIRValueUtil.isStackSlotValue;
import static org.graalvm.compiler.lir.LIRValueUtil.isVirtualStackSlot;
import static org.graalvm.compiler.lir.alloc.trace.TraceUtil.asShadowedRegisterValue;
import static org.graalvm.compiler.lir.alloc.trace.TraceUtil.isShadowedRegisterValue;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import org.graalvm.compiler.core.common.LIRKind;
import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig;
import org.graalvm.compiler.debug.CounterKey;
import org.graalvm.compiler.debug.DebugContext;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.debug.Indent;
import org.graalvm.compiler.lir.LIR;
import org.graalvm.compiler.lir.LIRInsertionBuffer;
import org.graalvm.compiler.lir.LIRInstruction;
import org.graalvm.compiler.lir.VirtualStackSlot;
import org.graalvm.compiler.lir.framemap.FrameMap;
import org.graalvm.compiler.lir.framemap.FrameMapBuilder;
import org.graalvm.compiler.lir.framemap.FrameMapBuilderTool;
import org.graalvm.compiler.lir.gen.LIRGenerationResult;
import org.graalvm.compiler.lir.gen.LIRGeneratorTool.MoveFactory;
import org.graalvm.compiler.options.OptionValues;
import jdk.vm.ci.code.Architecture;
import jdk.vm.ci.code.RegisterArray;
import jdk.vm.ci.code.StackSlot;
import jdk.vm.ci.meta.AllocatableValue;
import jdk.vm.ci.meta.Value;
public final class TraceGlobalMoveResolver extends TraceGlobalMoveResolutionPhase.MoveResolver {
private static final CounterKey cycleBreakingSlotsAllocated = DebugContext.counter("TraceRA[cycleBreakingSlotsAllocated(global)]");
private static final CounterKey cycleBreakingSlotsReused = DebugContext.counter("TraceRA[cycleBreakingSlotsReused(global)]");
private int insertIdx;
private LIRInsertionBuffer insertionBuffer;
private final ArrayList<Value> mappingFrom;
private final ArrayList<Value> mappingFromStack;
private final ArrayList<AllocatableValue> mappingTo;
private final int[] registerBlocked;
private static final int STACK_SLOT_IN_CALLER_FRAME_IDX = -1;
private int[] stackBlocked;
private final int firstVirtualStackIndex;
private final MoveFactory spillMoveFactory;
private final FrameMapBuilder frameMapBuilder;
private final OptionValues options;
private final RegisterAllocationConfig registerAllocationConfig;
private final LIRGenerationResult res;
private final DebugContext debug;
private void setValueBlocked(Value location, int direction) {
assert direction == 1 || direction == -1 : "out of bounds";
if (isStackSlotValue(location)) {
int stackIdx = getStackArrayIndex(location);
if (stackIdx == STACK_SLOT_IN_CALLER_FRAME_IDX) {
return;
}
if (stackIdx >= stackBlocked.length) {
stackBlocked = Arrays.copyOf(stackBlocked, stackIdx + 1);
}
stackBlocked[stackIdx] += direction;
} else {
assert direction == 1 || direction == -1 : "out of bounds";
if (isRegister(location)) {
registerBlocked[asRegister(location).number] += direction;
} else {
throw GraalError.shouldNotReachHere("unhandled value " + location);
}
}
}
private int valueBlocked(Value location) {
if (isStackSlotValue(location)) {
int stackIdx = getStackArrayIndex(location);
if (stackIdx == STACK_SLOT_IN_CALLER_FRAME_IDX) {
return 1;
}
if (stackIdx >= stackBlocked.length) {
return 0;
}
return stackBlocked[stackIdx];
}
if (isRegister(location)) {
return registerBlocked[asRegister(location).number];
}
throw GraalError.shouldNotReachHere("unhandled value " + location);
}
private static boolean areMultipleReadsAllowed() {
return true;
}
private boolean hasMappings() {
return mappingFrom.size() > 0;
}
private MoveFactory getSpillMoveFactory() {
return spillMoveFactory;
}
private RegisterArray getRegisters() {
return frameMapBuilder.getRegisterConfig().getAllocatableRegisters();
}
public TraceGlobalMoveResolver(LIRGenerationResult res, MoveFactory spillMoveFactory, RegisterAllocationConfig registerAllocationConfig, Architecture arch) {
this.mappingFrom = new ArrayList<>(8);
this.mappingFromStack = new ArrayList<>(8);
this.mappingTo = new ArrayList<>(8);
this.insertIdx = -1;
this.insertionBuffer = new LIRInsertionBuffer();
this.frameMapBuilder = res.getFrameMapBuilder();
this.spillMoveFactory = spillMoveFactory;
this.registerBlocked = new int[arch.getRegisters().size()];
this.registerAllocationConfig = registerAllocationConfig;
FrameMapBuilderTool frameMapBuilderTool = (FrameMapBuilderTool) frameMapBuilder;
this.stackBlocked = new int[frameMapBuilderTool.getNumberOfStackSlots()];
FrameMap frameMap = frameMapBuilderTool.getFrameMap();
this.firstVirtualStackIndex = !frameMap.frameNeedsAllocating() ? 0 : frameMap.currentFrameSize() + 1;
this.res = res;
LIR lir = res.getLIR();
this.options = lir.getOptions();
this.debug = lir.getDebug();
}
private boolean checkEmpty() {
for (int i = 0; i < stackBlocked.length; i++) {
assert stackBlocked[i] == 0 : "stack map must be empty before and after processing";
}
assert mappingFrom.size() == 0 && mappingTo.size() == 0 && mappingFromStack.size() == 0 : "list must be empty before and after processing";
for (int i = 0; i < getRegisters().size(); i++) {
assert registerBlocked[i] == 0 : "register map must be empty before and after processing";
}
return true;
}
private boolean verifyBeforeResolve() {
assert mappingFrom.size() == mappingTo.size() && mappingFrom.size() == mappingFromStack.size() : "length must be equal";
assert insertIdx != -1 : "insert position not set";
int i;
int j;
if (!areMultipleReadsAllowed()) {
for (i = 0; i < mappingFrom.size(); i++) {
for (j = i + 1; j < mappingFrom.size(); j++) {
assert mappingFrom.get(i) == null || mappingFrom.get(i) != mappingFrom.get(j) : "cannot read from same interval twice";
}
}
}
for (i = 0; i < mappingTo.size(); i++) {
for (j = i + 1; j < mappingTo.size(); j++) {
assert mappingTo.get(i) != mappingTo.get(j) : "cannot write to same interval twice";
}
}
for (i = 0; i < mappingTo.size(); i++) {
Value to = mappingTo.get(i);
assert !isStackSlotValue(to) || getStackArrayIndex(to) != STACK_SLOT_IN_CALLER_FRAME_IDX : "Cannot move to in argument: " + to;
}
HashSet<Value> usedRegs = new HashSet<>();
if (!areMultipleReadsAllowed()) {
for (i = 0; i < mappingFrom.size(); i++) {
Value from = mappingFrom.get(i);
if (from != null && !isIllegal(from)) {
boolean unique = usedRegs.add(from);
assert unique : "cannot read from same register twice";
}
}
}
usedRegs.clear();
for (i = 0; i < mappingTo.size(); i++) {
Value to = mappingTo.get(i);
if (isIllegal(to)) {
continue;
}
boolean unique = usedRegs.add(to);
assert unique : "cannot write to same register twice";
}
return true;
}
private void block(Value location) {
if (mightBeBlocked(location)) {
assert areMultipleReadsAllowed() || valueBlocked(location) == 0 : "location already marked as used: " + location;
setValueBlocked(location, 1);
debug.log("block %s", location);
}
}
private void unblock(Value location) {
if (mightBeBlocked(location)) {
assert valueBlocked(location) > 0 : "location already marked as unused: " + location;
setValueBlocked(location, -1);
debug.log("unblock %s", location);
}
}
private boolean safeToProcessMove(Value fromLocation, Value toLocation) {
if (mightBeBlocked(toLocation)) {
if ((valueBlocked(toLocation) > 1 || (valueBlocked(toLocation) == 1 && !isMoveToSelf(fromLocation, toLocation)))) {
return false;
}
}
return true;
}
public static boolean isMoveToSelf(Value from, Value to) {
assert to != null;
if (to.equals(from)) {
return true;
}
if (from == null) {
return false;
}
if (isShadowedRegisterValue(from)) {
if (isShadowedRegisterValue(to)) {
return false;
}
ShadowedRegisterValue shadowed = asShadowedRegisterValue(from);
if (isRegisterToRegisterMoveToSelf(shadowed.getRegister(), to)) {
return true;
}
if (isStackSlotValue(to)) {
return to.equals(shadowed.getStackSlot());
}
} else {
return isRegisterToRegisterMoveToSelf(from, to);
}
return false;
}
private static boolean isRegisterToRegisterMoveToSelf(Value from, Value to) {
if (to.equals(from)) {
return true;
}
if (isRegister(from) && isRegister(to) && asRegister(from).equals(asRegister(to))) {
return true;
}
return false;
}
private static boolean mightBeBlocked(Value location) {
return isRegister(location) || isStackSlotValue(location);
}
private void createInsertionBuffer(ArrayList<LIRInstruction> list) {
assert !insertionBuffer.initialized() : "overwriting existing buffer";
insertionBuffer.init(list);
}
private void appendInsertionBuffer() {
if (insertionBuffer.initialized()) {
insertionBuffer.finish();
}
assert !insertionBuffer.initialized() : "must be uninitialized now";
insertIdx = -1;
}
private LIRInstruction insertMove(Value fromOperand, AllocatableValue toOperand) {
assert !fromOperand.equals(toOperand) : "from and to are equal: " + fromOperand + " vs. " + toOperand;
assert LIRKind.verifyMoveKinds(fromOperand.getValueKind(), fromOperand.getValueKind(), registerAllocationConfig) : "move between different types";
assert insertIdx != -1 : "must setup insert position first";
LIRInstruction move = createMove(fromOperand, toOperand);
insertionBuffer.append(insertIdx, move);
if (debug.isLogEnabled()) {
debug.log("insert move from %s to %s at %d", fromOperand, toOperand, insertIdx);
}
return move;
}
private LIRInstruction createMove(Value fromOpr, AllocatableValue toOpr) {
if (isStackSlotValue(toOpr) && isStackSlotValue(fromOpr)) {
return getSpillMoveFactory().createStackMove(toOpr, asAllocatableValue(fromOpr));
}
return getSpillMoveFactory().createMove(toOpr, fromOpr);
}
@SuppressWarnings("try")
private void resolveMappings() {
try (Indent indent = debug.logAndIndent("resolveMapping")) {
assert verifyBeforeResolve();
if (debug.isLogEnabled()) {
printMapping();
}
for (int i = mappingFrom.size() - 1; i >= 0; i--) {
Value from = mappingFrom.get(i);
block(from);
}
ArrayList<AllocatableValue> busySpillSlots = null;
while (mappingFrom.size() > 0) {
boolean processedInterval = false;
int spillCandidate = -1;
for (int i = mappingFrom.size() - 1; i >= 0; i--) {
Value fromLocation = mappingFrom.get(i);
AllocatableValue toLocation = mappingTo.get(i);
if (safeToProcessMove(fromLocation, toLocation)) {
LIRInstruction move = insertMove(fromLocation, toLocation);
move.setComment(res, "TraceGlobalMoveResolver: resolveMapping");
unblock(fromLocation);
if (isStackSlotValue(toLocation)) {
if (busySpillSlots == null) {
busySpillSlots = new ArrayList<>(2);
}
busySpillSlots.add(toLocation);
}
mappingFrom.remove(i);
mappingFromStack.remove(i);
mappingTo.remove(i);
processedInterval = true;
} else if (fromLocation != null) {
if (isRegister(fromLocation) && (busySpillSlots == null || !busySpillSlots.contains(mappingFromStack.get(i)))) {
spillCandidate = i;
} else if (isStackSlotValue(fromLocation) && spillCandidate == -1) {
spillCandidate = i;
}
}
}
if (!processedInterval) {
breakCycle(spillCandidate);
}
}
}
assert checkEmpty();
}
@SuppressWarnings("try")
private void breakCycle(int spillCandidate) {
assert spillCandidate != -1 : "no interval in register for spilling found";
Value from = mappingFrom.get(spillCandidate);
try (Indent indent = debug.logAndIndent("BreakCycle: %s", from)) {
AllocatableValue spillSlot = null;
if (TraceRegisterAllocationPhase.Options.TraceRAreuseStackSlotsForMoveResolutionCycleBreaking.getValue(options) && !isStackSlotValue(from)) {
Value fromStack = mappingFromStack.get(spillCandidate);
if (fromStack != null) {
spillSlot = (AllocatableValue) fromStack;
cycleBreakingSlotsReused.increment(debug);
debug.log("reuse slot for spilling: %s", spillSlot);
}
}
if (spillSlot == null) {
spillSlot = frameMapBuilder.allocateSpillSlot(from.getValueKind());
cycleBreakingSlotsAllocated.increment(debug);
debug.log("created new slot for spilling: %s", spillSlot);
LIRInstruction move = insertMove(from, spillSlot);
move.setComment(res, "TraceGlobalMoveResolver: breakCycle");
}
block(spillSlot);
mappingFrom.set(spillCandidate, spillSlot);
unblock(from);
}
}
@SuppressWarnings("try")
private void printMapping() {
try (Indent indent = debug.logAndIndent("Mapping")) {
for (int i = mappingFrom.size() - 1; i >= 0; i--) {
debug.log("move %s <- %s (%s)", mappingTo.get(i), mappingFrom.get(i), mappingFromStack.get(i));
}
}
}
public void setInsertPosition(ArrayList<LIRInstruction> insertList, int insertIdx) {
assert this.insertIdx == -1 : "use moveInsertPosition instead of setInsertPosition when data already set";
createInsertionBuffer(insertList);
this.insertIdx = insertIdx;
}
@Override
public void addMapping(Value from, AllocatableValue to, Value fromStack) {
if (debug.isLogEnabled()) {
debug.log("add move mapping from %s to %s", from, to);
}
assert !from.equals(to) : "from and to interval equal: " + from;
assert LIRKind.verifyMoveKinds(to.getValueKind(), from.getValueKind(), registerAllocationConfig) : String.format("Kind mismatch: %s vs. %s, from=%s, to=%s", from.getValueKind(),
to.getValueKind(), from, to);
assert fromStack == null || LIRKind.verifyMoveKinds(to.getValueKind(), fromStack.getValueKind(), registerAllocationConfig) : String.format("Kind mismatch: %s vs. %s, fromStack=%s, to=%s",
fromStack.getValueKind(),
to.getValueKind(), fromStack, to);
mappingFrom.add(from);
mappingFromStack.add(fromStack);
mappingTo.add(to);
}
public void resolveAndAppendMoves() {
if (hasMappings()) {
resolveMappings();
}
appendInsertionBuffer();
}
private int getStackArrayIndex(Value stackSlotValue) {
if (isStackSlot(stackSlotValue)) {
return getStackArrayIndex(asStackSlot(stackSlotValue));
}
if (isVirtualStackSlot(stackSlotValue)) {
return getStackArrayIndex(asVirtualStackSlot(stackSlotValue));
}
throw GraalError.shouldNotReachHere("value is not a stack slot: " + stackSlotValue);
}
private int getStackArrayIndex(StackSlot stackSlot) {
int stackIdx;
if (stackSlot.isInCallerFrame()) {
stackIdx = STACK_SLOT_IN_CALLER_FRAME_IDX;
} else {
assert stackSlot.getRawAddFrameSize() : "Unexpected stack slot: " + stackSlot;
int offset = -stackSlot.getRawOffset();
assert 0 <= offset && offset < firstVirtualStackIndex : String.format("Wrong stack slot offset: %d (first virtual stack slot index: %d", offset, firstVirtualStackIndex);
stackIdx = offset;
}
return stackIdx;
}
private int getStackArrayIndex(VirtualStackSlot virtualStackSlot) {
return firstVirtualStackIndex + virtualStackSlot.getId();
}
}