package org.graalvm.compiler.lir.alloc.lsra;
import static jdk.vm.ci.code.CodeUtil.isEven;
import static jdk.vm.ci.code.ValueUtil.asRegister;
import static jdk.vm.ci.code.ValueUtil.isIllegal;
import static jdk.vm.ci.code.ValueUtil.isLegal;
import static jdk.vm.ci.code.ValueUtil.isRegister;
import static org.graalvm.compiler.lir.LIRValueUtil.isVariable;
import static org.graalvm.compiler.lir.phases.LIRPhase.Options.LIROptimization;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.EnumSet;
import jdk.internal.vm.compiler.collections.Pair;
import org.graalvm.compiler.core.common.LIRKind;
import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig;
import org.graalvm.compiler.core.common.cfg.AbstractBlockBase;
import org.graalvm.compiler.core.common.cfg.BlockMap;
import org.graalvm.compiler.debug.Assertions;
import org.graalvm.compiler.debug.DebugContext;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.debug.Indent;
import org.graalvm.compiler.lir.LIR;
import org.graalvm.compiler.lir.LIRInstruction;
import org.graalvm.compiler.lir.LIRInstruction.OperandFlag;
import org.graalvm.compiler.lir.LIRInstruction.OperandMode;
import org.graalvm.compiler.lir.ValueConsumer;
import org.graalvm.compiler.lir.Variable;
import org.graalvm.compiler.lir.VirtualStackSlot;
import org.graalvm.compiler.lir.alloc.lsra.Interval.RegisterBinding;
import org.graalvm.compiler.lir.framemap.FrameMapBuilder;
import org.graalvm.compiler.lir.gen.LIRGenerationResult;
import org.graalvm.compiler.lir.gen.LIRGeneratorTool.MoveFactory;
import org.graalvm.compiler.lir.phases.AllocationPhase.AllocationContext;
import org.graalvm.compiler.options.NestedBooleanOptionKey;
import org.graalvm.compiler.options.Option;
import org.graalvm.compiler.options.OptionKey;
import org.graalvm.compiler.options.OptionType;
import org.graalvm.compiler.options.OptionValues;
import jdk.vm.ci.code.Register;
import jdk.vm.ci.code.RegisterArray;
import jdk.vm.ci.code.RegisterAttributes;
import jdk.vm.ci.code.RegisterValue;
import jdk.vm.ci.code.TargetDescription;
import jdk.vm.ci.meta.AllocatableValue;
import jdk.vm.ci.meta.Value;
public class LinearScan {
public static class Options {
@Option(help = "Enable spill position optimization", type = OptionType.Debug)
public static final OptionKey<Boolean> LIROptLSRAOptimizeSpillPosition = new NestedBooleanOptionKey(LIROptimization, true);
}
public static class BlockData {
public BitSet liveIn;
public BitSet liveOut;
public BitSet liveGen;
public BitSet liveKill;
}
public static final int DOMINATOR_SPILL_MOVE_ID = -2;
private static final int SPLIT_INTERVALS_CAPACITY_RIGHT_SHIFT = 1;
private final LIR ir;
private final FrameMapBuilder frameMapBuilder;
private final RegisterAttributes[] registerAttributes;
private final RegisterArray registers;
private final RegisterAllocationConfig regAllocConfig;
private final MoveFactory moveFactory;
private final BlockMap<BlockData> blockData;
protected final DebugContext debug;
private final AbstractBlockBase<?>[] sortedBlocks;
private Interval[] intervals;
private int intervalsSize;
private int firstDerivedIntervalIndex = -1;
private Interval[] sortedIntervals;
private LIRInstruction[] opIdToInstructionMap;
private AbstractBlockBase<?>[] opIdToBlockMap;
private final int firstVariableNumber;
private int numVariables;
private final boolean neverSpillConstants;
protected final Interval intervalEndMarker;
public final Range rangeEndMarker;
public final boolean detailedAsserts;
private final LIRGenerationResult res;
protected LinearScan(TargetDescription target, LIRGenerationResult res, MoveFactory spillMoveFactory, RegisterAllocationConfig regAllocConfig, AbstractBlockBase<?>[] sortedBlocks,
boolean neverSpillConstants) {
this.ir = res.getLIR();
this.res = res;
this.debug = ir.getDebug();
this.moveFactory = spillMoveFactory;
this.frameMapBuilder = res.getFrameMapBuilder();
this.sortedBlocks = sortedBlocks;
this.registerAttributes = regAllocConfig.getRegisterConfig().getAttributesMap();
this.regAllocConfig = regAllocConfig;
this.registers = target.arch.getRegisters();
this.firstVariableNumber = getRegisters().size();
this.numVariables = ir.numVariables();
this.blockData = new BlockMap<>(ir.getControlFlowGraph());
this.neverSpillConstants = neverSpillConstants;
this.rangeEndMarker = new Range(Integer.MAX_VALUE, Integer.MAX_VALUE, null);
this.intervalEndMarker = new Interval(Value.ILLEGAL, Interval.END_MARKER_OPERAND_NUMBER, null, rangeEndMarker);
this.intervalEndMarker.next = intervalEndMarker;
this.detailedAsserts = Assertions.detailedAssertionsEnabled(ir.getOptions());
}
public LIRGenerationResult getLIRGenerationResult() {
return res;
}
public Interval intervalEndMarker() {
return intervalEndMarker;
}
public OptionValues getOptions() {
return ir.getOptions();
}
public DebugContext getDebug() {
return debug;
}
public int getFirstLirInstructionId(AbstractBlockBase<?> block) {
int result = ir.getLIRforBlock(block).get(0).id();
assert result >= 0;
return result;
}
public int getLastLirInstructionId(AbstractBlockBase<?> block) {
ArrayList<LIRInstruction> instructions = ir.getLIRforBlock(block);
int result = instructions.get(instructions.size() - 1).id();
assert result >= 0;
return result;
}
public MoveFactory getSpillMoveFactory() {
return moveFactory;
}
protected MoveResolver createMoveResolver() {
MoveResolver moveResolver = new MoveResolver(this);
assert moveResolver.checkEmpty();
return moveResolver;
}
public static boolean isVariableOrRegister(Value value) {
return isVariable(value) || isRegister(value);
}
int operandNumber(Value operand) {
if (isRegister(operand)) {
int number = asRegister(operand).number;
assert number < firstVariableNumber;
return number;
}
assert isVariable(operand) : operand;
return firstVariableNumber + ((Variable) operand).index;
}
int operandSize() {
return firstVariableNumber + numVariables;
}
int maxRegisterNumber() {
return firstVariableNumber - 1;
}
public BlockData getBlockData(AbstractBlockBase<?> block) {
return blockData.get(block);
}
void initBlockData(AbstractBlockBase<?> block) {
blockData.put(block, new BlockData());
}
static final IntervalPredicate IS_PRECOLORED_INTERVAL = new IntervalPredicate() {
@Override
public boolean apply(Interval i) {
return isRegister(i.operand);
}
};
static final IntervalPredicate IS_VARIABLE_INTERVAL = new IntervalPredicate() {
@Override
public boolean apply(Interval i) {
return isVariable(i.operand);
}
};
static final IntervalPredicate IS_STACK_INTERVAL = new IntervalPredicate() {
@Override
public boolean apply(Interval i) {
return !isRegister(i.operand);
}
};
public RegisterAttributes attributes(Register reg) {
return registerAttributes[reg.number];
}
void assignSpillSlot(Interval interval) {
if (interval.canMaterialize()) {
interval.assignLocation(Value.ILLEGAL);
} else if (interval.spillSlot() != null) {
interval.assignLocation(interval.spillSlot());
} else {
VirtualStackSlot slot = frameMapBuilder.allocateSpillSlot(interval.kind());
interval.setSpillSlot(slot);
interval.assignLocation(slot);
}
}
public Interval[] intervals() {
return intervals;
}
void initIntervals() {
intervalsSize = operandSize();
intervals = new Interval[intervalsSize + (intervalsSize >> SPLIT_INTERVALS_CAPACITY_RIGHT_SHIFT)];
}
Interval createInterval(AllocatableValue operand) {
assert isLegal(operand);
int operandNumber = operandNumber(operand);
Interval interval = new Interval(operand, operandNumber, intervalEndMarker, rangeEndMarker);
assert operandNumber < intervalsSize;
assert intervals[operandNumber] == null;
intervals[operandNumber] = interval;
return interval;
}
Interval createDerivedInterval(Interval source) {
if (firstDerivedIntervalIndex == -1) {
firstDerivedIntervalIndex = intervalsSize;
}
if (intervalsSize == intervals.length) {
intervals = Arrays.copyOf(intervals, intervals.length + (intervals.length >> SPLIT_INTERVALS_CAPACITY_RIGHT_SHIFT) + 1);
}
intervalsSize++;
assert intervalsSize <= intervals.length;
Variable variable = new Variable(source.kind(), numVariables++);
Interval interval = createInterval(variable);
assert intervals[intervalsSize - 1] == interval;
return interval;
}
public int blockCount() {
return sortedBlocks.length;
}
public AbstractBlockBase<?> blockAt(int index) {
return sortedBlocks[index];
}
public int liveSetSize() {
return firstDerivedIntervalIndex == -1 ? operandSize() : firstDerivedIntervalIndex;
}
int numLoops() {
return ir.getControlFlowGraph().getLoops().size();
}
Interval intervalFor(int operandNumber) {
return intervals[operandNumber];
}
public Interval intervalFor(Value operand) {
int operandNumber = operandNumber(operand);
assert operandNumber < intervalsSize;
return intervals[operandNumber];
}
public Interval getOrCreateInterval(AllocatableValue operand) {
Interval ret = intervalFor(operand);
if (ret == null) {
return createInterval(operand);
} else {
return ret;
}
}
void initOpIdMaps(int numInstructions) {
opIdToInstructionMap = new LIRInstruction[numInstructions];
opIdToBlockMap = new AbstractBlockBase<?>[numInstructions];
}
void putOpIdMaps(int index, LIRInstruction op, AbstractBlockBase<?> block) {
opIdToInstructionMap[index] = op;
opIdToBlockMap[index] = block;
}
int maxOpId() {
assert opIdToInstructionMap.length > 0 : "no operations";
return (opIdToInstructionMap.length - 1) << 1;
}
private static int opIdToIndex(int opId) {
return opId >> 1;
}
public LIRInstruction instructionForId(int opId) {
assert isEven(opId) : "opId not even";
LIRInstruction instr = opIdToInstructionMap[opIdToIndex(opId)];
assert instr.id() == opId;
return instr;
}
public AbstractBlockBase<?> blockForId(int opId) {
assert opIdToBlockMap.length > 0 && opId >= 0 && opId <= maxOpId() + 1 : "opId out of range";
return opIdToBlockMap[opIdToIndex(opId)];
}
boolean isBlockBegin(int opId) {
return opId == 0 || blockForId(opId) != blockForId(opId - 1);
}
boolean coversBlockBegin(int opId1, int opId2) {
return blockForId(opId1) != blockForId(opId2);
}
boolean hasCall(int opId) {
assert isEven(opId) : "opId not even";
return instructionForId(opId).destroysCallerSavedRegisters();
}
abstract static class IntervalPredicate {
abstract boolean apply(Interval i);
}
public boolean isProcessed(Value operand) {
return !isRegister(operand) || attributes(asRegister(operand)).isAllocatable();
}
private static boolean isSorted(Interval[] intervals) {
int from = -1;
for (Interval interval : intervals) {
assert interval != null;
assert from <= interval.from();
from = interval.from();
}
return true;
}
static Interval addToList(Interval first, Interval prev, Interval interval) {
Interval newFirst = first;
if (prev != null) {
prev.next = interval;
} else {
newFirst = interval;
}
return newFirst;
}
Pair<Interval, Interval> createUnhandledLists(IntervalPredicate isList1, IntervalPredicate isList2) {
assert isSorted(sortedIntervals) : "interval list is not sorted";
Interval list1 = intervalEndMarker;
Interval list2 = intervalEndMarker;
Interval list1Prev = null;
Interval list2Prev = null;
Interval v;
int n = sortedIntervals.length;
for (int i = 0; i < n; i++) {
v = sortedIntervals[i];
if (v == null) {
continue;
}
if (isList1.apply(v)) {
list1 = addToList(list1, list1Prev, v);
list1Prev = v;
} else if (isList2 == null || isList2.apply(v)) {
list2 = addToList(list2, list2Prev, v);
list2Prev = v;
}
}
if (list1Prev != null) {
list1Prev.next = intervalEndMarker;
}
if (list2Prev != null) {
list2Prev.next = intervalEndMarker;
}
assert list1Prev == null || list1Prev.next.isEndMarker() : "linear list ends not with sentinel";
assert list2Prev == null || list2Prev.next.isEndMarker() : "linear list ends not with sentinel";
return Pair.create(list1, list2);
}
protected void sortIntervalsBeforeAllocation() {
int sortedLen = 0;
for (Interval interval : intervals) {
if (interval != null) {
sortedLen++;
}
}
Interval[] sortedList = new Interval[sortedLen];
int sortedIdx = 0;
int sortedFromMax = -1;
for (Interval interval : intervals) {
if (interval != null) {
int from = interval.from();
if (sortedFromMax <= from) {
sortedList[sortedIdx++] = interval;
sortedFromMax = interval.from();
} else {
int j;
for (j = sortedIdx - 1; j >= 0 && from < sortedList[j].from(); j--) {
sortedList[j + 1] = sortedList[j];
}
sortedList[j + 1] = interval;
sortedIdx++;
}
}
}
sortedIntervals = sortedList;
}
void sortIntervalsAfterAllocation() {
if (firstDerivedIntervalIndex == -1) {
return;
}
Interval[] oldList = sortedIntervals;
Interval[] newList = Arrays.copyOfRange(intervals, firstDerivedIntervalIndex, intervalsSize);
int oldLen = oldList.length;
int newLen = newList.length;
Arrays.sort(newList, (Interval a, Interval b) -> a.from() - b.from());
Interval[] combinedList = new Interval[oldLen + newLen];
int oldIdx = 0;
int newIdx = 0;
while (oldIdx + newIdx < combinedList.length) {
if (newIdx >= newLen || (oldIdx < oldLen && oldList[oldIdx].from() <= newList[newIdx].from())) {
combinedList[oldIdx + newIdx] = oldList[oldIdx];
oldIdx++;
} else {
combinedList[oldIdx + newIdx] = newList[newIdx];
newIdx++;
}
}
sortedIntervals = combinedList;
}
public Interval splitChildAtOpId(Interval interval, int opId, LIRInstruction.OperandMode mode) {
Interval result = interval.getSplitChildAtOpId(opId, mode, this);
if (result != null) {
if (debug.isLogEnabled()) {
debug.log("Split child at pos %d of interval %s is %s", opId, interval, result);
}
return result;
}
throw new GraalError("LinearScan: interval is null");
}
static AllocatableValue canonicalSpillOpr(Interval interval) {
assert interval.spillSlot() != null : "canonical spill slot not set";
return interval.spillSlot();
}
boolean isMaterialized(AllocatableValue operand, int opId, OperandMode mode) {
Interval interval = intervalFor(operand);
assert interval != null : "interval must exist";
if (opId != -1) {
interval = splitChildAtOpId(interval, opId, mode);
}
return isIllegal(interval.location()) && interval.canMaterialize();
}
boolean isCallerSave(Value operand) {
return attributes(asRegister(operand)).isCallerSave();
}
@SuppressWarnings("try")
protected void allocate(TargetDescription target, LIRGenerationResult lirGenRes, AllocationContext context) {
try (Indent indent = debug.logAndIndent("LinearScan allocate")) {
createLifetimeAnalysisPhase().apply(target, lirGenRes, context);
try (DebugContext.Scope s = debug.scope("AfterLifetimeAnalysis", (Object) intervals)) {
sortIntervalsBeforeAllocation();
createRegisterAllocationPhase().apply(target, lirGenRes, context);
if (LinearScan.Options.LIROptLSRAOptimizeSpillPosition.getValue(getOptions())) {
createOptimizeSpillPositionPhase().apply(target, lirGenRes, context);
}
createResolveDataFlowPhase().apply(target, lirGenRes, context);
sortIntervalsAfterAllocation();
if (detailedAsserts) {
verify();
}
beforeSpillMoveElimination();
createSpillMoveEliminationPhase().apply(target, lirGenRes, context);
createAssignLocationsPhase().apply(target, lirGenRes, context);
if (detailedAsserts) {
verifyIntervals();
}
} catch (Throwable e) {
throw debug.handle(e);
}
}
}
protected void beforeSpillMoveElimination() {
}
protected LinearScanLifetimeAnalysisPhase createLifetimeAnalysisPhase() {
return new LinearScanLifetimeAnalysisPhase(this);
}
protected LinearScanRegisterAllocationPhase createRegisterAllocationPhase() {
return new LinearScanRegisterAllocationPhase(this);
}
protected LinearScanOptimizeSpillPositionPhase createOptimizeSpillPositionPhase() {
return new LinearScanOptimizeSpillPositionPhase(this);
}
protected LinearScanResolveDataFlowPhase createResolveDataFlowPhase() {
return new LinearScanResolveDataFlowPhase(this);
}
protected LinearScanEliminateSpillMovePhase createSpillMoveEliminationPhase() {
return new LinearScanEliminateSpillMovePhase(this);
}
protected LinearScanAssignLocationsPhase createAssignLocationsPhase() {
return new LinearScanAssignLocationsPhase(this);
}
@SuppressWarnings("try")
public void printIntervals(String label) {
if (debug.isLogEnabled()) {
try (Indent indent = debug.logAndIndent("intervals %s", label)) {
for (Interval interval : intervals) {
if (interval != null) {
debug.log("%s", interval.logString(this));
}
}
try (Indent indent2 = debug.logAndIndent("Basic Blocks")) {
for (int i = 0; i < blockCount(); i++) {
AbstractBlockBase<?> block = blockAt(i);
debug.log("B%d [%d, %d, %s] ", block.getId(), getFirstLirInstructionId(block), getLastLirInstructionId(block), block.getLoop());
}
}
}
}
debug.dump(DebugContext.VERBOSE_LEVEL, new LinearScanIntervalDumper(Arrays.copyOf(intervals, intervalsSize)), label);
}
boolean verify() {
verifyIntervals();
verifyRegisters();
debug.log("no errors found");
return true;
}
@SuppressWarnings("try")
private void verifyRegisters() {
try (Indent indent = debug.logAndIndent("verifying register allocation")) {
RegisterVerifier verifier = new RegisterVerifier(this);
verifier.verify(blockAt(0));
}
}
@SuppressWarnings("try")
protected void verifyIntervals() {
try (Indent indent = debug.logAndIndent("verifying intervals")) {
int len = intervalsSize;
for (int i = 0; i < len; i++) {
Interval i1 = intervals[i];
if (i1 == null) {
continue;
}
i1.checkSplitChildren();
if (i1.operandNumber != i) {
debug.log("Interval %d is on position %d in list", i1.operandNumber, i);
debug.log(i1.logString(this));
throw new GraalError("");
}
if (isVariable(i1.operand) && i1.kind().equals(LIRKind.Illegal)) {
debug.log("Interval %d has no type assigned", i1.operandNumber);
debug.log(i1.logString(this));
throw new GraalError("");
}
if (i1.location() == null) {
debug.log("Interval %d has no register assigned", i1.operandNumber);
debug.log(i1.logString(this));
throw new GraalError("");
}
if (i1.first().isEndMarker()) {
debug.log("Interval %d has no Range", i1.operandNumber);
debug.log(i1.logString(this));
throw new GraalError("");
}
for (Range r = i1.first(); !r.isEndMarker(); r = r.next) {
if (r.from >= r.to) {
debug.log("Interval %d has zero length range", i1.operandNumber);
debug.log(i1.logString(this));
throw new GraalError("");
}
}
for (int j = i + 1; j < len; j++) {
Interval i2 = intervals[j];
if (i2 == null) {
continue;
}
if (i1.from() == 1 && i1.to() == 2) {
continue;
}
if (i2.from() == 1 && i2.to() == 2) {
continue;
}
Value l1 = i1.location();
Value l2 = i2.location();
if (i1.intersects(i2) && !isIllegal(l1) && (l1.equals(l2))) {
throw GraalError.shouldNotReachHere(String.format("Intervals %d and %d overlap and have the same register assigned\n%s\n%s", i1.operandNumber, i2.operandNumber,
i1.logString(this), i2.logString(this)));
}
}
}
}
}
class CheckConsumer implements ValueConsumer {
boolean ok;
Interval curInterval;
@Override
public void visitValue(Value operand, OperandMode mode, EnumSet<OperandFlag> flags) {
if (isRegister(operand)) {
if (intervalFor(operand) == curInterval) {
ok = true;
}
}
}
}
@SuppressWarnings("try")
void verifyNoOopsInFixedIntervals() {
try (Indent indent = debug.logAndIndent("verifying that no oops are in fixed intervals *")) {
CheckConsumer checkConsumer = new CheckConsumer();
Interval fixedIntervals;
Interval otherIntervals;
fixedIntervals = createUnhandledLists(IS_PRECOLORED_INTERVAL, null).getLeft();
otherIntervals = new Interval(Value.ILLEGAL, -1, intervalEndMarker, rangeEndMarker);
otherIntervals.addRange(Integer.MAX_VALUE - 2, Integer.MAX_VALUE - 1);
IntervalWalker iw = new IntervalWalker(this, fixedIntervals, otherIntervals);
for (AbstractBlockBase<?> block : sortedBlocks) {
ArrayList<LIRInstruction> instructions = ir.getLIRforBlock(block);
for (int j = 0; j < instructions.size(); j++) {
LIRInstruction op = instructions.get(j);
if (op.hasState()) {
iw.walkBefore(op.id());
boolean checkLive = true;
if (checkLive) {
for (Interval interval = iw.activeLists.get(RegisterBinding.Fixed); !interval.isEndMarker(); interval = interval.next) {
if (interval.currentTo() > op.id() + 1) {
checkConsumer.curInterval = interval;
checkConsumer.ok = false;
op.visitEachInput(checkConsumer);
op.visitEachAlive(checkConsumer);
op.visitEachTemp(checkConsumer);
op.visitEachOutput(checkConsumer);
assert checkConsumer.ok : "fixed intervals should never be live across an oopmap point";
}
}
}
}
}
}
}
}
public LIR getLIR() {
return ir;
}
public FrameMapBuilder getFrameMapBuilder() {
return frameMapBuilder;
}
public AbstractBlockBase<?>[] sortedBlocks() {
return sortedBlocks;
}
public RegisterArray getRegisters() {
return registers;
}
public RegisterAllocationConfig getRegisterAllocationConfig() {
return regAllocConfig;
}
public boolean callKillsRegisters() {
return regAllocConfig.getRegisterConfig().areAllAllocatableRegistersCallerSaved();
}
boolean neverSpillConstants() {
return neverSpillConstants;
}
}