package org.graalvm.compiler.lir.asm;
import static jdk.vm.ci.code.ValueUtil.asStackSlot;
import static jdk.vm.ci.code.ValueUtil.isStackSlot;
import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.Consumer;
import jdk.internal.vm.compiler.collections.EconomicMap;
import jdk.internal.vm.compiler.collections.Equivalence;
import org.graalvm.compiler.asm.AbstractAddress;
import org.graalvm.compiler.asm.Assembler;
import org.graalvm.compiler.asm.Label;
import org.graalvm.compiler.code.CompilationResult;
import org.graalvm.compiler.code.CompilationResult.CodeAnnotation;
import org.graalvm.compiler.code.CompilationResult.JumpTable;
import org.graalvm.compiler.code.DataSection.Data;
import org.graalvm.compiler.code.DataSection.RawData;
import org.graalvm.compiler.core.common.NumUtil;
import org.graalvm.compiler.core.common.cfg.AbstractBlockBase;
import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
import org.graalvm.compiler.core.common.type.DataPointerConstant;
import org.graalvm.compiler.debug.DebugContext;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.graph.NodeSourcePosition;
import org.graalvm.compiler.lir.LIR;
import org.graalvm.compiler.lir.LIRFrameState;
import org.graalvm.compiler.lir.LIRInstruction;
import org.graalvm.compiler.lir.LIRInstructionVerifier;
import org.graalvm.compiler.lir.LabelRef;
import org.graalvm.compiler.lir.StandardOp.LabelHoldingOp;
import org.graalvm.compiler.lir.framemap.FrameMap;
import org.graalvm.compiler.options.Option;
import org.graalvm.compiler.options.OptionKey;
import org.graalvm.compiler.options.OptionType;
import org.graalvm.compiler.options.OptionValues;
import org.graalvm.compiler.serviceprovider.GraalServices;
import jdk.vm.ci.code.BailoutException;
import jdk.vm.ci.code.CodeCacheProvider;
import jdk.vm.ci.code.DebugInfo;
import jdk.vm.ci.code.Register;
import jdk.vm.ci.code.StackSlot;
import jdk.vm.ci.code.TargetDescription;
import jdk.vm.ci.code.site.Call;
import jdk.vm.ci.code.site.ConstantReference;
import jdk.vm.ci.code.site.DataSectionReference;
import jdk.vm.ci.code.site.Infopoint;
import jdk.vm.ci.code.site.InfopointReason;
import jdk.vm.ci.meta.Constant;
import jdk.vm.ci.meta.InvokeTarget;
import jdk.vm.ci.meta.JavaConstant;
import jdk.vm.ci.meta.JavaKind;
import jdk.vm.ci.meta.VMConstant;
import jdk.vm.ci.meta.Value;
public class CompilationResultBuilder {
private static final List<LIRInstructionVerifier> LIR_INSTRUCTION_VERIFIERS = new ArrayList<>();
static {
for (LIRInstructionVerifier verifier : GraalServices.load(LIRInstructionVerifier.class)) {
if (verifier.isEnabled()) {
LIR_INSTRUCTION_VERIFIERS.add(verifier);
}
}
}
public static class Options {
@Option(help = "Include the LIR as comments with the final assembly.", type = OptionType.Debug)
public static final OptionKey<Boolean> PrintLIRWithAssembly = new OptionKey<>(false);
}
private static class ExceptionInfo {
public final int codeOffset;
public final LabelRef exceptionEdge;
ExceptionInfo(int pcOffset, LabelRef exceptionEdge) {
this.codeOffset = pcOffset;
this.exceptionEdge = exceptionEdge;
}
}
public static final class AssemblerAnnotation extends CodeAnnotation {
public final Assembler.CodeAnnotation assemblerCodeAnnotation;
public AssemblerAnnotation(Assembler.CodeAnnotation assemblerCodeAnnotation) {
super(assemblerCodeAnnotation.instructionPosition);
this.assemblerCodeAnnotation = assemblerCodeAnnotation;
}
@Override
public boolean equals(Object obj) {
return this == obj;
}
@Override
public String toString() {
return assemblerCodeAnnotation.toString();
}
}
public final Assembler asm;
public final DataBuilder dataBuilder;
public final CompilationResult compilationResult;
public final Register uncompressedNullRegister;
public final TargetDescription target;
public final CodeCacheProvider codeCache;
public final ForeignCallsProvider foreignCalls;
public final FrameMap frameMap;
protected LIR lir;
protected int currentBlockIndex;
public final FrameContext frameContext;
private List<ExceptionInfo> exceptionInfoList;
private final OptionValues options;
private final DebugContext debug;
private final EconomicMap<Constant, Data> dataCache;
private Consumer<LIRInstruction> beforeOp;
private Consumer<LIRInstruction> afterOp;
private EconomicMap<Label, Integer> labelBindLirPositions;
private EconomicMap<LIRInstruction, Integer> lirPositions;
private boolean conservativeLabelOffsets = false;
public final boolean mustReplaceWithUncompressedNullRegister(JavaConstant nullConstant) {
return !uncompressedNullRegister.equals(Register.None) && JavaConstant.NULL_POINTER.equals(nullConstant);
}
private boolean needsMHDeoptHandler = false;
public CompilationResultBuilder(CodeCacheProvider codeCache,
ForeignCallsProvider foreignCalls,
FrameMap frameMap,
Assembler asm,
DataBuilder dataBuilder,
FrameContext frameContext,
OptionValues options,
DebugContext debug,
CompilationResult compilationResult,
Register uncompressedNullRegister) {
this(codeCache,
foreignCalls,
frameMap,
asm,
dataBuilder,
frameContext,
options,
debug,
compilationResult,
uncompressedNullRegister,
EconomicMap.create(Equivalence.DEFAULT));
}
public CompilationResultBuilder(CodeCacheProvider codeCache,
ForeignCallsProvider foreignCalls,
FrameMap frameMap,
Assembler asm,
DataBuilder dataBuilder,
FrameContext frameContext,
OptionValues options,
DebugContext debug,
CompilationResult compilationResult,
Register uncompressedNullRegister,
EconomicMap<Constant, Data> dataCache) {
this.target = codeCache.getTarget();
this.codeCache = codeCache;
this.foreignCalls = foreignCalls;
this.frameMap = frameMap;
this.asm = asm;
this.dataBuilder = dataBuilder;
this.compilationResult = compilationResult;
this.uncompressedNullRegister = uncompressedNullRegister;
this.frameContext = frameContext;
this.options = options;
this.debug = debug;
assert frameContext != null;
this.dataCache = dataCache;
}
public void setTotalFrameSize(int frameSize) {
compilationResult.setTotalFrameSize(frameSize);
}
public void setMaxInterpreterFrameSize(int maxInterpreterFrameSize) {
compilationResult.setMaxInterpreterFrameSize(maxInterpreterFrameSize);
}
public CompilationResult.CodeMark recordMark(CompilationResult.MarkId id) {
CompilationResult.CodeMark mark = compilationResult.recordMark(asm.position(), id);
if (currentCallContext != null) {
currentCallContext.recordMark(mark);
}
return mark;
}
public void (String s) {
compilationResult.addAnnotation(new CompilationResult.CodeComment(asm.position(), s));
}
public void finish() {
int position = asm.position();
compilationResult.setTargetCode(asm.close(false), position);
if (exceptionInfoList != null) {
for (ExceptionInfo ei : exceptionInfoList) {
int codeOffset = ei.codeOffset;
compilationResult.recordExceptionHandler(codeOffset, ei.exceptionEdge.label().position());
}
}
closeCompilationResult();
}
protected void closeCompilationResult() {
compilationResult.close();
}
public void recordExceptionHandlers(int pcOffset, LIRFrameState info) {
if (info != null) {
if (info.exceptionEdge != null) {
if (exceptionInfoList == null) {
exceptionInfoList = new ArrayList<>(4);
}
exceptionInfoList.add(new ExceptionInfo(pcOffset, info.exceptionEdge));
}
}
}
public void recordImplicitException(int pcOffset, LIRFrameState info) {
compilationResult.recordInfopoint(pcOffset, info.debugInfo(), InfopointReason.IMPLICIT_EXCEPTION);
assert info.exceptionEdge == null;
}
public boolean isImplicitExceptionExist(int pcOffset) {
List<Infopoint> infopoints = compilationResult.getInfopoints();
for (Infopoint infopoint : infopoints) {
if (infopoint.pcOffset == pcOffset && infopoint.reason == InfopointReason.IMPLICIT_EXCEPTION) {
return true;
}
}
return false;
}
public void recordDirectCall(int posBefore, int posAfter, InvokeTarget callTarget, LIRFrameState info) {
DebugInfo debugInfo = info != null ? info.debugInfo() : null;
Call call = compilationResult.recordCall(posBefore, posAfter - posBefore, callTarget, debugInfo, true);
if (currentCallContext != null) {
currentCallContext.recordCall(call);
}
}
public void recordIndirectCall(int posBefore, int posAfter, InvokeTarget callTarget, LIRFrameState info) {
DebugInfo debugInfo = info != null ? info.debugInfo() : null;
compilationResult.recordCall(posBefore, posAfter - posBefore, callTarget, debugInfo, false);
}
public void recordInfopoint(int pos, LIRFrameState info, InfopointReason reason) {
DebugInfo debugInfo = info.debugInfo();
recordInfopoint(pos, debugInfo, reason);
}
public void recordInfopoint(int pos, DebugInfo debugInfo, InfopointReason reason) {
compilationResult.recordInfopoint(pos, debugInfo, reason);
}
public void recordSourceMapping(int pcOffset, int endPcOffset, NodeSourcePosition sourcePosition) {
compilationResult.recordSourceMapping(pcOffset, endPcOffset, sourcePosition);
}
public void recordInlineDataInCode(Constant data) {
assert data != null;
int pos = asm.position();
debug.log("Inline data in code: pos = %d, data = %s", pos, data);
if (data instanceof VMConstant) {
compilationResult.recordDataPatch(pos, new ConstantReference((VMConstant) data));
}
}
public void recordInlineDataInCodeWithNote(Constant data, Object note) {
assert data != null;
int pos = asm.position();
debug.log("Inline data in code: pos = %d, data = %s, note = %s", pos, data, note);
if (data instanceof VMConstant) {
compilationResult.recordDataPatchWithNote(pos, new ConstantReference((VMConstant) data), note);
}
}
public AbstractAddress recordDataSectionReference(Data data) {
assert data != null;
DataSectionReference reference = compilationResult.getDataSection().insertData(data);
int instructionStart = asm.position();
compilationResult.recordDataPatch(instructionStart, reference);
return asm.getPlaceholder(instructionStart);
}
public AbstractAddress recordDataReferenceInCode(DataPointerConstant constant) {
return recordDataReferenceInCode(constant, constant.getAlignment());
}
public AbstractAddress recordDataReferenceInCode(Constant constant, int alignment) {
assert constant != null;
debug.log("Constant reference in code: pos = %d, data = %s", asm.position(), constant);
Data data = createDataItem(constant);
data.updateAlignment(alignment);
return recordDataSectionReference(data);
}
public AbstractAddress recordDataReferenceInCode(Data data, int alignment) {
assert data != null;
data.updateAlignment(alignment);
return recordDataSectionReference(data);
}
public Data createDataItem(Constant constant) {
Data data = dataCache.get(constant);
if (data == null) {
data = dataBuilder.createDataItem(constant);
dataCache.put(constant, data);
}
return data;
}
public AbstractAddress recordDataReferenceInCode(byte[] data, int alignment) {
assert data != null;
if (debug.isLogEnabled()) {
debug.log("Data reference in code: pos = %d, data = %s", asm.position(), Arrays.toString(data));
}
return recordDataSectionReference(new RawData(data, alignment));
}
public int asIntConst(Value value) {
assert isJavaConstant(value) && asJavaConstant(value).getJavaKind().isNumericInteger();
JavaConstant constant = asJavaConstant(value);
long c = constant.asLong();
if (!NumUtil.isInt(c)) {
throw GraalError.shouldNotReachHere();
}
return (int) c;
}
public float asFloatConst(Value value) {
assert isJavaConstant(value) && asJavaConstant(value).getJavaKind() == JavaKind.Float;
JavaConstant constant = asJavaConstant(value);
return constant.asFloat();
}
public long asLongConst(Value value) {
assert isJavaConstant(value) && asJavaConstant(value).getJavaKind() == JavaKind.Long;
JavaConstant constant = asJavaConstant(value);
return constant.asLong();
}
public double asDoubleConst(Value value) {
assert isJavaConstant(value) && asJavaConstant(value).getJavaKind() == JavaKind.Double;
JavaConstant constant = asJavaConstant(value);
return constant.asDouble();
}
public AbstractAddress asFloatConstRef(JavaConstant value) {
return asFloatConstRef(value, 4);
}
public AbstractAddress asFloatConstRef(JavaConstant value, int alignment) {
assert value.getJavaKind() == JavaKind.Float;
return recordDataReferenceInCode(value, alignment);
}
public AbstractAddress asDoubleConstRef(JavaConstant value) {
return asDoubleConstRef(value, 8);
}
public AbstractAddress asDoubleConstRef(JavaConstant value, int alignment) {
assert value.getJavaKind() == JavaKind.Double;
return recordDataReferenceInCode(value, alignment);
}
public AbstractAddress asLongConstRef(JavaConstant value) {
assert value.getJavaKind() == JavaKind.Long;
return recordDataReferenceInCode(value, 8);
}
public AbstractAddress asObjectConstRef(JavaConstant value) {
assert value.getJavaKind() == JavaKind.Object;
return recordDataReferenceInCode(value, 8);
}
public AbstractAddress asByteAddr(Value value) {
assert value.getPlatformKind().getSizeInBytes() >= JavaKind.Byte.getByteCount();
return asAddress(value);
}
public AbstractAddress asShortAddr(Value value) {
assert value.getPlatformKind().getSizeInBytes() >= JavaKind.Short.getByteCount();
return asAddress(value);
}
public AbstractAddress asIntAddr(Value value) {
assert value.getPlatformKind().getSizeInBytes() >= JavaKind.Int.getByteCount();
return asAddress(value);
}
public AbstractAddress asLongAddr(Value value) {
assert value.getPlatformKind().getSizeInBytes() >= JavaKind.Long.getByteCount();
return asAddress(value);
}
public AbstractAddress asFloatAddr(Value value) {
assert value.getPlatformKind().getSizeInBytes() >= JavaKind.Float.getByteCount();
return asAddress(value);
}
public AbstractAddress asDoubleAddr(Value value) {
assert value.getPlatformKind().getSizeInBytes() >= JavaKind.Double.getByteCount();
return asAddress(value);
}
public AbstractAddress asAddress(Value value) {
assert isStackSlot(value);
StackSlot slot = asStackSlot(value);
return asm.makeAddress(frameMap.getRegisterConfig().getFrameRegister(), frameMap.offsetForStackSlot(slot));
}
public boolean isSuccessorEdge(LabelRef edge) {
assert lir != null;
AbstractBlockBase<?>[] order = lir.codeEmittingOrder();
assert order[currentBlockIndex] == edge.getSourceBlock();
AbstractBlockBase<?> nextBlock = LIR.getNextBlock(order, currentBlockIndex);
return nextBlock == edge.getTargetBlock();
}
public void emit(@SuppressWarnings("hiding") LIR lir) {
assert this.lir == null;
assert currentBlockIndex == 0;
this.lir = lir;
this.currentBlockIndex = 0;
frameContext.enter(this);
for (AbstractBlockBase<?> b : lir.codeEmittingOrder()) {
assert (b == null && lir.codeEmittingOrder()[currentBlockIndex] == null) || lir.codeEmittingOrder()[currentBlockIndex].equals(b);
emitBlock(b);
currentBlockIndex++;
}
this.lir = null;
this.currentBlockIndex = 0;
}
private void emitBlock(AbstractBlockBase<?> block) {
if (block == null) {
return;
}
boolean emitComment = debug.isDumpEnabled(DebugContext.BASIC_LEVEL) || Options.PrintLIRWithAssembly.getValue(getOptions());
if (emitComment) {
blockComment(String.format("block B%d %s", block.getId(), block.getLoop()));
}
for (LIRInstruction op : lir.getLIRforBlock(block)) {
if (emitComment) {
blockComment(String.format("%d %s", op.id(), op));
}
try {
if (beforeOp != null) {
beforeOp.accept(op);
}
emitOp(op);
if (afterOp != null) {
afterOp.accept(op);
}
} catch (GraalError e) {
throw e.addContext("lir instruction", block + "@" + op.id() + " " + op.getClass().getName() + " " + op + "\n" + Arrays.toString(lir.codeEmittingOrder()));
}
}
}
private void emitOp(LIRInstruction op) {
try {
int start = asm.position();
op.emitCode(this);
if (op.getPosition() != null) {
recordSourceMapping(start, asm.position(), op.getPosition());
}
if (LIR_INSTRUCTION_VERIFIERS.size() > 0 && start < asm.position()) {
int end = asm.position();
for (CodeAnnotation codeAnnotation : compilationResult.getCodeAnnotations()) {
if (codeAnnotation instanceof JumpTable) {
int jumpTableStart = codeAnnotation.getPosition();
if (jumpTableStart >= start && jumpTableStart < end) {
end = jumpTableStart;
}
}
}
byte[] emittedCode = asm.copy(start, end);
for (LIRInstructionVerifier verifier : LIR_INSTRUCTION_VERIFIERS) {
verifier.verify(op, emittedCode);
}
}
} catch (BailoutException e) {
throw e;
} catch (AssertionError t) {
throw new GraalError(t);
} catch (RuntimeException t) {
throw new GraalError(t);
}
}
public void resetForEmittingCode() {
asm.reset();
compilationResult.resetForEmittingCode();
if (exceptionInfoList != null) {
exceptionInfoList.clear();
}
if (dataCache != null) {
dataCache.clear();
}
lir = null;
currentBlockIndex = 0;
}
public void setOpCallback(Consumer<LIRInstruction> beforeOp, Consumer<LIRInstruction> afterOp) {
this.beforeOp = beforeOp;
this.afterOp = afterOp;
}
public OptionValues getOptions() {
return options;
}
public void buildLabelOffsets(LIR generatedLIR) {
labelBindLirPositions = EconomicMap.create(Equivalence.IDENTITY);
lirPositions = EconomicMap.create(Equivalence.IDENTITY);
int instructionPosition = 0;
for (AbstractBlockBase<?> block : generatedLIR.codeEmittingOrder()) {
if (block != null) {
for (LIRInstruction op : generatedLIR.getLIRforBlock(block)) {
if (op instanceof LabelHoldingOp) {
Label label = ((LabelHoldingOp) op).getLabel();
if (label != null) {
labelBindLirPositions.put(label, instructionPosition);
}
}
lirPositions.put(op, instructionPosition);
instructionPosition++;
}
}
}
}
public boolean labelWithinRange(LIRInstruction instruction, Label label, int disp) {
if (conservativeLabelOffsets) {
return false;
}
Integer labelPosition = labelBindLirPositions.get(label);
Integer instructionPosition = lirPositions.get(instruction);
boolean result;
if (labelPosition != null && instructionPosition != null) {
result = Math.abs(labelPosition - instructionPosition) < disp;
} else {
result = false;
}
return result;
}
public void setConservativeLabelRanges() {
this.conservativeLabelOffsets = true;
}
public final boolean needsClearUpperVectorRegisters() {
for (AbstractBlockBase<?> block : lir.codeEmittingOrder()) {
if (block == null) {
continue;
}
for (LIRInstruction op : lir.getLIRforBlock(block)) {
if (op.needsClearUpperVectorRegisters()) {
return true;
}
}
}
return false;
}
private CallContext currentCallContext;
public final class CallContext implements AutoCloseable {
private CompilationResult.CodeMark mark;
private Call call;
@Override
public void close() {
currentCallContext = null;
compilationResult.recordCallContext(mark, call);
}
void recordCall(Call c) {
assert this.call == null : "Recording call twice";
this.call = c;
}
void recordMark(CompilationResult.CodeMark m) {
assert this.mark == null : "Recording mark twice";
this.mark = m;
}
}
public CallContext openCallContext(boolean direct) {
if (currentCallContext != null) {
throw GraalError.shouldNotReachHere("Call context already open");
}
if (compilationResult.isImmutablePIC() && direct) {
currentCallContext = new CallContext();
}
return currentCallContext;
}
public void setNeedsMHDeoptHandler() {
this.needsMHDeoptHandler = true;
}
public boolean needsMHDeoptHandler() {
return needsMHDeoptHandler;
}
}