package org.graalvm.compiler.lir.alloc.lsra;
import static jdk.vm.ci.code.CodeUtil.isOdd;
import static jdk.vm.ci.code.ValueUtil.asRegister;
import static jdk.vm.ci.code.ValueUtil.isRegister;
import static org.graalvm.compiler.lir.LIRValueUtil.isStackSlotValue;
import static org.graalvm.compiler.lir.LIRValueUtil.isVariable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig.AllocatableRegisters;
import org.graalvm.compiler.core.common.cfg.AbstractBlockBase;
import org.graalvm.compiler.core.common.util.Util;
import org.graalvm.compiler.debug.DebugContext;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.debug.Indent;
import org.graalvm.compiler.lir.LIRInstruction;
import org.graalvm.compiler.lir.StandardOp.ValueMoveOp;
import org.graalvm.compiler.lir.alloc.OutOfRegistersException;
import org.graalvm.compiler.lir.alloc.lsra.Interval.RegisterBinding;
import org.graalvm.compiler.lir.alloc.lsra.Interval.RegisterPriority;
import org.graalvm.compiler.lir.alloc.lsra.Interval.SpillState;
import org.graalvm.compiler.lir.alloc.lsra.Interval.State;
import jdk.vm.ci.code.Register;
import jdk.vm.ci.meta.Value;
class LinearScanWalker extends IntervalWalker {
protected Register[] availableRegs;
protected final int[] usePos;
protected final int[] blockPos;
protected List<Interval>[] spillIntervals;
private MoveResolver moveResolver;
private int minReg;
private int maxReg;
private static final List<Interval> EMPTY_LIST = Collections.emptyList();
int blockCount() {
return allocator.blockCount();
}
AbstractBlockBase<?> blockAt(int idx) {
return allocator.blockAt(idx);
}
AbstractBlockBase<?> blockOfOpWithId(int opId) {
return allocator.blockForId(opId);
}
LinearScanWalker(LinearScan allocator, Interval unhandledFixedFirst, Interval unhandledAnyFirst) {
super(allocator, unhandledFixedFirst, unhandledAnyFirst);
moveResolver = allocator.createMoveResolver();
spillIntervals = Util.uncheckedCast(new List<?>[allocator.getRegisters().size()]);
for (int i = 0; i < allocator.getRegisters().size(); i++) {
spillIntervals[i] = EMPTY_LIST;
}
usePos = new int[allocator.getRegisters().size()];
blockPos = new int[allocator.getRegisters().size()];
}
void initUseLists(boolean onlyProcessUsePos) {
for (Register register : availableRegs) {
int i = register.number;
usePos[i] = Integer.MAX_VALUE;
if (!onlyProcessUsePos) {
blockPos[i] = Integer.MAX_VALUE;
spillIntervals[i].clear();
}
}
}
int maxRegisterNumber() {
return maxReg;
}
int minRegisterNumber() {
return minReg;
}
boolean isRegisterInRange(int reg) {
return reg >= minRegisterNumber() && reg <= maxRegisterNumber();
}
void excludeFromUse(Interval i) {
Value location = i.location();
int i1 = asRegister(location).number;
if (isRegisterInRange(i1)) {
usePos[i1] = 0;
}
}
void setUsePos(Interval interval, int usePos, boolean onlyProcessUsePos) {
if (usePos != -1) {
assert usePos != 0 : "must use excludeFromUse to set usePos to 0";
int i = asRegister(interval.location()).number;
if (isRegisterInRange(i)) {
if (this.usePos[i] > usePos) {
this.usePos[i] = usePos;
}
if (!onlyProcessUsePos) {
List<Interval> list = spillIntervals[i];
if (list == EMPTY_LIST) {
list = new ArrayList<>(2);
spillIntervals[i] = list;
}
list.add(interval);
}
}
}
}
void setBlockPos(Interval i, int blockPos) {
if (blockPos != -1) {
int reg = asRegister(i.location()).number;
if (isRegisterInRange(reg)) {
if (this.blockPos[reg] > blockPos) {
this.blockPos[reg] = blockPos;
}
if (usePos[reg] > blockPos) {
usePos[reg] = blockPos;
}
}
}
}
void freeExcludeActiveFixed() {
Interval interval = activeLists.get(RegisterBinding.Fixed);
while (!interval.isEndMarker()) {
assert isRegister(interval.location()) : "active interval must have a register assigned";
excludeFromUse(interval);
interval = interval.next;
}
}
void freeExcludeActiveAny() {
Interval interval = activeLists.get(RegisterBinding.Any);
while (!interval.isEndMarker()) {
assert isRegister(interval.location()) : "active interval must have a register assigned";
excludeFromUse(interval);
interval = interval.next;
}
}
void freeCollectInactiveFixed(Interval current) {
Interval interval = inactiveLists.get(RegisterBinding.Fixed);
while (!interval.isEndMarker()) {
if (current.to() <= interval.currentFrom()) {
assert interval.currentIntersectsAt(current) == -1 : "must not intersect";
setUsePos(interval, interval.currentFrom(), true);
} else {
setUsePos(interval, interval.currentIntersectsAt(current), true);
}
interval = interval.next;
}
}
void freeCollectInactiveAny(Interval current) {
Interval interval = inactiveLists.get(RegisterBinding.Any);
while (!interval.isEndMarker()) {
setUsePos(interval, interval.currentIntersectsAt(current), true);
interval = interval.next;
}
}
void freeCollectUnhandled(RegisterBinding kind, Interval current) {
Interval interval = unhandledLists.get(kind);
while (!interval.isEndMarker()) {
setUsePos(interval, interval.intersectsAt(current), true);
if (kind == RegisterBinding.Fixed && current.to() <= interval.from()) {
setUsePos(interval, interval.from(), true);
}
interval = interval.next;
}
}
void spillExcludeActiveFixed() {
Interval interval = activeLists.get(RegisterBinding.Fixed);
while (!interval.isEndMarker()) {
excludeFromUse(interval);
interval = interval.next;
}
}
void spillBlockUnhandledFixed(Interval current) {
Interval interval = unhandledLists.get(RegisterBinding.Fixed);
while (!interval.isEndMarker()) {
setBlockPos(interval, interval.intersectsAt(current));
interval = interval.next;
}
}
void spillBlockInactiveFixed(Interval current) {
Interval interval = inactiveLists.get(RegisterBinding.Fixed);
while (!interval.isEndMarker()) {
if (current.to() > interval.currentFrom()) {
setBlockPos(interval, interval.currentIntersectsAt(current));
} else {
assert interval.currentIntersectsAt(current) == -1 : "invalid optimization: intervals intersect";
}
interval = interval.next;
}
}
void spillCollectActiveAny(RegisterPriority registerPriority) {
Interval interval = activeLists.get(RegisterBinding.Any);
while (!interval.isEndMarker()) {
setUsePos(interval, Math.min(interval.nextUsage(registerPriority, currentPosition), interval.to()), false);
interval = interval.next;
}
}
void spillCollectInactiveAny(Interval current) {
Interval interval = inactiveLists.get(RegisterBinding.Any);
while (!interval.isEndMarker()) {
if (interval.currentIntersects(current)) {
setUsePos(interval, Math.min(interval.nextUsage(RegisterPriority.LiveAtLoopEnd, currentPosition), interval.to()), false);
}
interval = interval.next;
}
}
void insertMove(int operandId, Interval srcIt, Interval dstIt) {
int opId = (operandId + 1) & ~1;
AbstractBlockBase<?> opBlock = allocator.blockForId(opId);
assert opId > 0 && allocator.blockForId(opId - 2) == opBlock : "cannot insert move at block boundary";
ArrayList<LIRInstruction> instructions = allocator.getLIR().getLIRforBlock(opBlock);
int index = (opId - instructions.get(0).id()) >> 1;
assert instructions.get(index).id() <= opId : "error in calculation";
while (instructions.get(index).id() != opId) {
index++;
assert 0 <= index && index < instructions.size() : "index out of bounds";
}
assert 1 <= index && index < instructions.size() : "index out of bounds";
assert instructions.get(index).id() == opId : "error in calculation";
moveResolver.moveInsertPosition(instructions, index);
moveResolver.addMapping(srcIt, dstIt);
}
int findOptimalSplitPos(AbstractBlockBase<?> minBlock, AbstractBlockBase<?> maxBlock, int maxSplitPos) {
int fromBlockNr = minBlock.getLinearScanNumber();
int toBlockNr = maxBlock.getLinearScanNumber();
assert 0 <= fromBlockNr && fromBlockNr < blockCount() : "out of range";
assert 0 <= toBlockNr && toBlockNr < blockCount() : "out of range";
assert fromBlockNr < toBlockNr : "must cross block boundary";
int optimalSplitPos = allocator.getLastLirInstructionId(maxBlock) + 2;
if (optimalSplitPos > maxSplitPos) {
optimalSplitPos = allocator.getFirstLirInstructionId(maxBlock);
}
int minLoopDepth = maxBlock.getLoopDepth();
for (int i = toBlockNr - 1; minLoopDepth > 0 && i >= fromBlockNr; i--) {
AbstractBlockBase<?> cur = blockAt(i);
if (cur.getLoopDepth() < minLoopDepth) {
minLoopDepth = cur.getLoopDepth();
optimalSplitPos = allocator.getLastLirInstructionId(cur) + 2;
}
}
assert optimalSplitPos > allocator.maxOpId() || allocator.isBlockBegin(optimalSplitPos) : "algorithm must move split pos to block boundary";
return optimalSplitPos;
}
int findOptimalSplitPos(Interval interval, int minSplitPos, int maxSplitPos, boolean doLoopOptimization) {
DebugContext debug = allocator.getDebug();
int optimalSplitPos = -1;
if (minSplitPos == maxSplitPos) {
if (debug.isLogEnabled()) {
debug.log("min-pos and max-pos are equal, no optimization possible");
}
optimalSplitPos = minSplitPos;
} else {
assert minSplitPos < maxSplitPos : "must be true then";
assert minSplitPos > 0 : "cannot access minSplitPos - 1 otherwise";
AbstractBlockBase<?> minBlock = allocator.blockForId(minSplitPos - 1);
AbstractBlockBase<?> maxBlock = allocator.blockForId(maxSplitPos - 1);
assert minBlock.getLinearScanNumber() <= maxBlock.getLinearScanNumber() : "invalid order";
if (minBlock == maxBlock) {
if (debug.isLogEnabled()) {
debug.log("cannot move split pos to block boundary because minPos and maxPos are in same block");
}
optimalSplitPos = maxSplitPos;
} else {
if (interval.hasHoleBetween(maxSplitPos - 1, maxSplitPos) && !allocator.isBlockBegin(maxSplitPos)) {
if (debug.isLogEnabled()) {
debug.log("interval has hole just before maxSplitPos, so splitting at maxSplitPos");
}
optimalSplitPos = maxSplitPos;
} else {
if (debug.isLogEnabled()) {
debug.log("moving split pos to optimal block boundary between block B%d and B%d", minBlock.getId(), maxBlock.getId());
}
if (doLoopOptimization) {
int loopEndPos = interval.nextUsageExact(RegisterPriority.LiveAtLoopEnd, allocator.getLastLirInstructionId(minBlock) + 2);
if (debug.isLogEnabled()) {
debug.log("loop optimization: loop end found at pos %d", loopEndPos);
}
assert loopEndPos > minSplitPos : "invalid order";
if (loopEndPos < maxSplitPos) {
AbstractBlockBase<?> loopBlock = allocator.blockForId(loopEndPos);
if (debug.isLogEnabled()) {
debug.log("interval is used in loop that ends in block B%d, so trying to move maxBlock back from B%d to B%d", loopBlock.getId(), maxBlock.getId(), loopBlock.getId());
}
assert loopBlock != minBlock : "loopBlock and minBlock must be different because block boundary is needed between";
int maxSpillPos = allocator.getLastLirInstructionId(loopBlock) + 2;
optimalSplitPos = findOptimalSplitPos(minBlock, loopBlock, maxSpillPos);
if (optimalSplitPos == maxSpillPos) {
optimalSplitPos = -1;
if (debug.isLogEnabled()) {
debug.log("loop optimization not necessary");
}
} else {
if (debug.isLogEnabled()) {
debug.log("loop optimization successful");
}
}
}
}
if (optimalSplitPos == -1) {
optimalSplitPos = findOptimalSplitPos(minBlock, maxBlock, maxSplitPos);
}
}
}
}
if (debug.isLogEnabled()) {
debug.log("optimal split position: %d", optimalSplitPos);
}
return optimalSplitPos;
}
@SuppressWarnings("try")
void splitBeforeUsage(Interval interval, int minSplitPos, int maxSplitPos) {
DebugContext debug = allocator.getDebug();
try (Indent indent = debug.logAndIndent("splitting interval %s between %d and %d", interval, minSplitPos, maxSplitPos)) {
assert interval.from() < minSplitPos : "cannot split at start of interval";
assert currentPosition < minSplitPos : "cannot split before current position";
assert minSplitPos <= maxSplitPos : "invalid order";
assert maxSplitPos <= interval.to() : "cannot split after end of interval";
int optimalSplitPos = findOptimalSplitPos(interval, minSplitPos, maxSplitPos, true);
assert minSplitPos <= optimalSplitPos && optimalSplitPos <= maxSplitPos : "out of range";
assert optimalSplitPos <= interval.to() : "cannot split after end of interval";
assert optimalSplitPos > interval.from() : "cannot split at start of interval";
if (optimalSplitPos == interval.to() && interval.nextUsage(RegisterPriority.MustHaveRegister, minSplitPos) == Integer.MAX_VALUE) {
if (debug.isLogEnabled()) {
debug.log("no split necessary because optimal split position is at end of interval");
}
return;
}
boolean moveNecessary = !allocator.isBlockBegin(optimalSplitPos) && !interval.hasHoleBetween(optimalSplitPos - 1, optimalSplitPos);
if (!allocator.isBlockBegin(optimalSplitPos)) {
optimalSplitPos = (optimalSplitPos - 1) | 1;
}
if (debug.isLogEnabled()) {
debug.log("splitting at position %d", optimalSplitPos);
}
assert allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 1) : "split pos must be odd when not on block boundary";
assert !allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 0) : "split pos must be even on block boundary";
Interval splitPart = interval.split(optimalSplitPos, allocator);
splitPart.setInsertMoveWhenActivated(moveNecessary);
assert splitPart.from() >= currentPosition : "cannot append new interval before current walk position";
unhandledLists.addToListSortedByStartAndUsePositions(RegisterBinding.Any, splitPart);
if (debug.isLogEnabled()) {
debug.log("left interval %s: %s", moveNecessary ? " " : "", interval.logString(allocator));
debug.log("right interval %s: %s", moveNecessary ? "(move)" : "", splitPart.logString(allocator));
}
}
}
@SuppressWarnings("try")
void splitForSpilling(Interval interval) {
DebugContext debug = allocator.getDebug();
int maxSplitPos = currentPosition;
int previousUsage = interval.previousUsage(RegisterPriority.ShouldHaveRegister, maxSplitPos);
if (previousUsage == currentPosition) {
previousUsage = interval.previousUsage(RegisterPriority.MustHaveRegister, maxSplitPos);
}
int minSplitPos = Math.max(previousUsage + 1, interval.from());
try (Indent indent = debug.logAndIndent("splitting and spilling interval %s between %d and %d", interval, minSplitPos, maxSplitPos)) {
assert interval.state == State.Active : "why spill interval that is not active?";
assert interval.from() <= minSplitPos : "cannot split before start of interval";
assert minSplitPos <= maxSplitPos : "invalid order";
assert maxSplitPos < interval.to() : "cannot split at end end of interval";
assert currentPosition < interval.to() : "interval must not end before current position";
if (minSplitPos == interval.from()) {
try (Indent indent2 = debug.logAndIndent("spilling entire interval because split pos is at beginning of interval (use positions: %d)", interval.usePosList().size())) {
assert interval.firstUsage(RegisterPriority.MustHaveRegister) > currentPosition : String.format("interval %s must not have use position before currentPosition %d", interval,
currentPosition);
allocator.assignSpillSlot(interval);
handleSpillSlot(interval);
changeSpillState(interval, minSplitPos);
Interval parent = interval;
while (parent != null && parent.isSplitChild()) {
parent = parent.getSplitChildBeforeOpId(parent.from());
if (isRegister(parent.location())) {
if (parent.firstUsage(RegisterPriority.ShouldHaveRegister) == Integer.MAX_VALUE) {
if (debug.isLogEnabled()) {
debug.log("kicking out interval %d out of its register because it is never used", parent.operandNumber);
}
allocator.assignSpillSlot(parent);
handleSpillSlot(parent);
} else {
parent = null;
}
}
}
}
} else {
int optimalSplitPos = findOptimalSplitPos(interval, minSplitPos, maxSplitPos, false);
assert minSplitPos <= optimalSplitPos && optimalSplitPos <= maxSplitPos : "out of range";
assert optimalSplitPos < interval.to() : "cannot split at end of interval";
assert optimalSplitPos >= interval.from() : "cannot split before start of interval";
if (!allocator.isBlockBegin(optimalSplitPos)) {
optimalSplitPos = (optimalSplitPos - 1) | 1;
}
try (Indent indent2 = debug.logAndIndent("splitting at position %d", optimalSplitPos)) {
assert allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 1) : "split pos must be odd when not on block boundary";
assert !allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 0) : "split pos must be even on block boundary";
Interval spilledPart = interval.split(optimalSplitPos, allocator);
allocator.assignSpillSlot(spilledPart);
handleSpillSlot(spilledPart);
changeSpillState(spilledPart, optimalSplitPos);
if (!allocator.isBlockBegin(optimalSplitPos)) {
if (debug.isLogEnabled()) {
debug.log("inserting move from interval %d to %d", interval.operandNumber, spilledPart.operandNumber);
}
insertMove(optimalSplitPos, interval, spilledPart);
}
assert spilledPart.currentSplitChild() == interval : "overwriting wrong currentSplitChild";
spilledPart.makeCurrentSplitChild();
if (debug.isLogEnabled()) {
debug.log("left interval: %s", interval.logString(allocator));
debug.log("spilled interval : %s", spilledPart.logString(allocator));
}
}
}
}
}
private void changeSpillState(Interval interval, int spillPos) {
switch (interval.spillState()) {
case NoSpillStore: {
int defLoopDepth = allocator.blockForId(interval.spillDefinitionPos()).getLoopDepth();
int spillLoopDepth = allocator.blockForId(spillPos).getLoopDepth();
if (defLoopDepth < spillLoopDepth) {
if (LinearScan.Options.LIROptLSRAOptimizeSpillPosition.getValue(allocator.getOptions())) {
interval.setSpillState(SpillState.SpillInDominator);
} else {
interval.setSpillState(SpillState.StoreAtDefinition);
}
} else {
interval.setSpillState(SpillState.OneSpillStore);
}
break;
}
case OneSpillStore: {
int defLoopDepth = allocator.blockForId(interval.spillDefinitionPos()).getLoopDepth();
int spillLoopDepth = allocator.blockForId(spillPos).getLoopDepth();
if (defLoopDepth <= spillLoopDepth) {
if (LinearScan.Options.LIROptLSRAOptimizeSpillPosition.getValue(allocator.getOptions())) {
interval.setSpillState(SpillState.SpillInDominator);
} else {
interval.setSpillState(SpillState.StoreAtDefinition);
}
}
break;
}
case SpillInDominator:
case StoreAtDefinition:
case StartInMemory:
case NoOptimization:
case NoDefinitionFound:
break;
default:
throw GraalError.shouldNotReachHere("other states not allowed at this time");
}
}
protected void handleSpillSlot(Interval interval) {
assert interval.location() != null && (interval.canMaterialize() || isStackSlotValue(interval.location())) : "interval not assigned to a stack slot " + interval;
}
void splitStackInterval(Interval interval) {
int minSplitPos = currentPosition + 1;
int maxSplitPos = Math.min(interval.firstUsage(RegisterPriority.ShouldHaveRegister), interval.to());
splitBeforeUsage(interval, minSplitPos, maxSplitPos);
}
void splitWhenPartialRegisterAvailable(Interval interval, int registerAvailableUntil) {
int minSplitPos = Math.max(interval.previousUsage(RegisterPriority.ShouldHaveRegister, registerAvailableUntil), interval.from() + 1);
splitBeforeUsage(interval, minSplitPos, registerAvailableUntil);
}
void splitAndSpillInterval(Interval interval) {
assert interval.state == State.Active || interval.state == State.Inactive : "other states not allowed";
int currentPos = currentPosition;
if (interval.state == State.Inactive) {
assert interval.hasHoleBetween(currentPos - 1, currentPos + 1) : "interval can not be inactive otherwise";
splitBeforeUsage(interval, currentPos + 1, currentPos + 1);
} else {
int minSplitPos = currentPos + 1;
int maxSplitPos = Math.min(interval.nextUsage(RegisterPriority.MustHaveRegister, minSplitPos), interval.to());
splitBeforeUsage(interval, minSplitPos, maxSplitPos);
assert interval.nextUsage(RegisterPriority.MustHaveRegister, currentPos) == Integer.MAX_VALUE : "the remaining part is spilled to stack and therefore has no register";
splitForSpilling(interval);
}
}
@SuppressWarnings("try")
boolean allocFreeRegister(Interval interval) {
DebugContext debug = allocator.getDebug();
try (Indent indent = debug.logAndIndent("trying to find free register for %s", interval)) {
initUseLists(true);
freeExcludeActiveFixed();
freeExcludeActiveAny();
freeCollectInactiveFixed(interval);
freeCollectInactiveAny(interval);
assert unhandledLists.get(RegisterBinding.Fixed).isEndMarker() : "must not have unhandled fixed intervals because all fixed intervals have a use at position 0";
if (debug.isLogEnabled()) {
try (Indent indent2 = debug.logAndIndent("state of registers:")) {
for (Register register : availableRegs) {
int i = register.number;
debug.log("reg %d: usePos: %d", register.number, usePos[i]);
}
}
}
Register hint = null;
Interval locationHint = interval.locationHint(true);
if (locationHint != null && locationHint.location() != null && isRegister(locationHint.location())) {
hint = asRegister(locationHint.location());
if (debug.isLogEnabled()) {
debug.log("hint register %d from interval %s", hint.number, locationHint);
}
}
assert interval.location() == null : "register already assigned to interval";
int regNeededUntil = interval.from() + 1;
int intervalTo = interval.to();
boolean needSplit = false;
int splitPos = -1;
Register reg = null;
Register minFullReg = null;
Register maxPartialReg = null;
for (Register availableReg : availableRegs) {
int number = availableReg.number;
if (usePos[number] >= intervalTo) {
if (minFullReg == null || availableReg.equals(hint) || (usePos[number] < usePos[minFullReg.number] && !minFullReg.equals(hint))) {
minFullReg = availableReg;
}
} else if (usePos[number] > regNeededUntil) {
if (maxPartialReg == null || availableReg.equals(hint) || (usePos[number] > usePos[maxPartialReg.number] && !maxPartialReg.equals(hint))) {
maxPartialReg = availableReg;
}
}
}
if (minFullReg != null) {
reg = minFullReg;
} else if (maxPartialReg != null) {
needSplit = true;
reg = maxPartialReg;
} else {
return false;
}
splitPos = usePos[reg.number];
interval.assignLocation(reg.asValue(interval.kind()));
if (debug.isLogEnabled()) {
debug.log("selected register %d", reg.number);
}
assert splitPos > 0 : "invalid splitPos";
if (needSplit) {
splitWhenPartialRegisterAvailable(interval, splitPos);
}
return true;
}
}
void splitAndSpillIntersectingIntervals(Register reg) {
assert reg != null : "no register assigned";
for (int i = 0; i < spillIntervals[reg.number].size(); i++) {
Interval interval = spillIntervals[reg.number].get(i);
removeFromList(interval);
splitAndSpillInterval(interval);
}
}
@SuppressWarnings("try")
void allocLockedRegister(Interval interval) {
DebugContext debug = allocator.getDebug();
try (Indent indent = debug.logAndIndent("alloc locked register: need to split and spill to get register for %s", interval)) {
int firstUsage = interval.firstUsage(RegisterPriority.MustHaveRegister);
int firstShouldHaveUsage = interval.firstUsage(RegisterPriority.ShouldHaveRegister);
int regNeededUntil = Math.min(firstUsage, interval.from() + 1);
int intervalTo = interval.to();
assert regNeededUntil >= 0 && regNeededUntil < Integer.MAX_VALUE : "interval has no use";
Register reg;
Register ignore;
for (RegisterPriority registerPriority = RegisterPriority.LiveAtLoopEnd; true; registerPriority = RegisterPriority.MustHaveRegister) {
initUseLists(false);
spillExcludeActiveFixed();
assert unhandledLists.get(RegisterBinding.Fixed).isEndMarker() : "must not have unhandled fixed intervals because all fixed intervals have a use at position 0";
spillBlockInactiveFixed(interval);
spillCollectActiveAny(registerPriority);
spillCollectInactiveAny(interval);
if (debug.isLogEnabled()) {
printRegisterState();
}
reg = null;
ignore = interval.location() != null && isRegister(interval.location()) ? asRegister(interval.location()) : null;
for (Register availableReg : availableRegs) {
int number = availableReg.number;
if (availableReg.equals(ignore)) {
} else if (usePos[number] > regNeededUntil) {
if (reg == null || (usePos[number] > usePos[reg.number])) {
reg = availableReg;
}
}
}
int regUsePos = (reg == null ? 0 : usePos[reg.number]);
if (regUsePos <= firstShouldHaveUsage) {
if (debug.isLogEnabled()) {
debug.log("able to spill current interval. firstUsage(register): %d, usePos: %d", firstUsage, regUsePos);
}
if (firstUsage <= interval.from() + 1) {
if (registerPriority.equals(RegisterPriority.LiveAtLoopEnd)) {
debug.log("retry with register priority must have register");
continue;
}
String description = generateOutOfRegErrorMsg(interval, firstUsage, availableRegs);
allocator.assignSpillSlot(interval);
debug.dump(DebugContext.INFO_LEVEL, allocator.getLIR(), description);
allocator.printIntervals(description);
throw new OutOfRegistersException("LinearScan: no register found", description);
}
splitAndSpillInterval(interval);
return;
}
break;
}
boolean needSplit = blockPos[reg.number] <= intervalTo;
int splitPos = blockPos[reg.number];
if (debug.isLogEnabled()) {
debug.log("decided to use register %d", reg.number);
}
assert splitPos > 0 : "invalid splitPos";
assert needSplit || splitPos > interval.from() : "splitting interval at from";
interval.assignLocation(reg.asValue(interval.kind()));
if (needSplit) {
splitWhenPartialRegisterAvailable(interval, splitPos);
}
splitAndSpillIntersectingIntervals(reg);
return;
}
}
private static String generateOutOfRegErrorMsg(Interval interval, int firstUsage, Register[] availableRegs) {
return "Cannot spill interval (" + interval + ") that is used in first instruction (possible reason: no register found) firstUsage=" + firstUsage +
", interval.from()=" + interval.from() + "; already used candidates: " + Arrays.toString(availableRegs);
}
@SuppressWarnings("try")
void printRegisterState() {
DebugContext debug = allocator.getDebug();
try (Indent indent2 = debug.logAndIndent("state of registers:")) {
for (Register reg : availableRegs) {
int i = reg.number;
try (Indent indent3 = debug.logAndIndent("reg %d: usePos: %d, blockPos: %d, intervals: ", i, usePos[i], blockPos[i])) {
for (int j = 0; j < spillIntervals[i].size(); j++) {
debug.log("%s ", spillIntervals[i].get(j));
}
}
}
}
}
boolean noAllocationPossible(Interval interval) {
if (allocator.callKillsRegisters()) {
int pos = interval.from();
if (isOdd(pos)) {
if (pos < allocator.maxOpId() && allocator.hasCall(pos + 1) && interval.to() > pos + 1) {
DebugContext debug = allocator.getDebug();
if (debug.isLogEnabled()) {
debug.log("free register cannot be available because all registers blocked by following call");
}
assert !allocFreeRegister(interval) : "found a register for this interval";
return true;
}
}
}
return false;
}
void initVarsForAlloc(Interval interval) {
AllocatableRegisters allocatableRegisters = allocator.getRegisterAllocationConfig().getAllocatableRegisters(interval.kind().getPlatformKind());
availableRegs = allocatableRegisters.allocatableRegisters;
minReg = allocatableRegisters.minRegisterNumber;
maxReg = allocatableRegisters.maxRegisterNumber;
}
static boolean isMove(LIRInstruction op, Interval from, Interval to) {
if (ValueMoveOp.isValueMoveOp(op)) {
ValueMoveOp move = ValueMoveOp.asValueMoveOp(op);
if (isVariable(move.getInput()) && isVariable(move.getResult())) {
return move.getInput() != null && move.getInput().equals(from.operand) && move.getResult() != null && move.getResult().equals(to.operand);
}
}
return false;
}
void combineSpilledIntervals(Interval interval) {
if (interval.isSplitChild()) {
return;
}
Interval registerHint = interval.locationHint(false);
if (registerHint == null) {
return;
}
assert registerHint.isSplitParent() : "register hint must be split parent";
if (interval.spillState() != SpillState.NoOptimization || registerHint.spillState() != SpillState.NoOptimization) {
return;
}
int beginPos = interval.from();
int endPos = interval.to();
if (endPos > allocator.maxOpId() || isOdd(beginPos) || isOdd(endPos)) {
return;
}
if (!isMove(allocator.instructionForId(beginPos), registerHint, interval) || !isMove(allocator.instructionForId(endPos), interval, registerHint)) {
return;
}
Interval beginHint = registerHint.getSplitChildAtOpId(beginPos, LIRInstruction.OperandMode.USE, allocator);
Interval endHint = registerHint.getSplitChildAtOpId(endPos, LIRInstruction.OperandMode.DEF, allocator);
if (beginHint == endHint || beginHint.to() != beginPos || endHint.from() != endPos) {
return;
}
assert beginHint.location() != null : "must have register assigned";
assert endHint.location() == null : "must not have register assigned";
assert interval.firstUsage(RegisterPriority.MustHaveRegister) == beginPos : "must have use position at begin of interval because of move";
assert endHint.firstUsage(RegisterPriority.MustHaveRegister) == endPos : "must have use position at begin of interval because of move";
if (isRegister(beginHint.location())) {
return;
}
assert registerHint.spillSlot() != null : "must be set when part of interval was spilled";
interval.setSpillSlot(registerHint.spillSlot());
interval.removeFirstUsePos();
endHint.removeFirstUsePos();
}
@Override
@SuppressWarnings("try")
protected boolean activateCurrent(Interval interval) {
boolean result = true;
DebugContext debug = allocator.getDebug();
try (Indent indent = debug.logAndIndent("activating interval %s, splitParent: %d", interval, interval.splitParent().operandNumber)) {
final Value operand = interval.operand;
if (interval.location() != null && isStackSlotValue(interval.location())) {
if (debug.isLogEnabled()) {
debug.log("interval has spill slot assigned (method parameter) . split it before first use");
}
splitStackInterval(interval);
result = false;
} else {
if (interval.location() == null) {
if (debug.isLogEnabled()) {
debug.log("normal allocation of register");
}
combineSpilledIntervals(interval);
initVarsForAlloc(interval);
if (noAllocationPossible(interval) || !allocFreeRegister(interval)) {
allocLockedRegister(interval);
}
if (!isRegister(interval.location())) {
result = false;
}
}
}
if (interval.insertMoveWhenActivated()) {
assert interval.isSplitChild();
assert interval.currentSplitChild() != null;
assert !interval.currentSplitChild().operand.equals(operand) : "cannot insert move between same interval";
if (debug.isLogEnabled()) {
debug.log("Inserting move from interval %d to %d because insertMoveWhenActivated is set", interval.currentSplitChild().operandNumber, interval.operandNumber);
}
insertMove(interval.from(), interval.currentSplitChild(), interval);
}
interval.makeCurrentSplitChild();
}
return result;
}
public void finishAllocation() {
moveResolver.resolveAndAppendMoves();
}
}