package org.graalvm.compiler.core.aarch64;
import static jdk.vm.ci.aarch64.AArch64.sp;
import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
import static org.graalvm.compiler.lir.LIRValueUtil.isIntConstant;
import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant;
import java.util.function.Function;
import org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode;
import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ConditionFlag;
import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
import org.graalvm.compiler.core.common.LIRKind;
import org.graalvm.compiler.core.common.calc.Condition;
import org.graalvm.compiler.core.common.spi.LIRKindTool;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.lir.LIRFrameState;
import org.graalvm.compiler.lir.LIRValueUtil;
import org.graalvm.compiler.lir.LabelRef;
import org.graalvm.compiler.lir.StandardOp;
import org.graalvm.compiler.lir.SwitchStrategy;
import org.graalvm.compiler.lir.Variable;
import org.graalvm.compiler.lir.aarch64.AArch64AddressValue;
import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp;
import org.graalvm.compiler.lir.aarch64.AArch64ArrayCompareToOp;
import org.graalvm.compiler.lir.aarch64.AArch64ArrayEqualsOp;
import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddLSEOp;
import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddOp;
import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndWriteOp;
import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.CompareAndSwapOp;
import org.graalvm.compiler.lir.aarch64.AArch64ByteSwapOp;
import org.graalvm.compiler.lir.aarch64.AArch64Compare;
import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow;
import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.BranchOp;
import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CompareBranchZeroOp;
import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondMoveOp;
import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondSetOp;
import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.StrategySwitchOp;
import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.TableSwitchOp;
import org.graalvm.compiler.lir.aarch64.AArch64LIRFlagsVersioned;
import org.graalvm.compiler.lir.aarch64.AArch64Move;
import org.graalvm.compiler.lir.aarch64.AArch64Move.MembarOp;
import org.graalvm.compiler.lir.aarch64.AArch64PauseOp;
import org.graalvm.compiler.lir.aarch64.AArch64SpeculativeBarrier;
import org.graalvm.compiler.lir.aarch64.AArch64ZeroMemoryOp;
import org.graalvm.compiler.lir.gen.LIRGenerationResult;
import org.graalvm.compiler.lir.gen.LIRGenerator;
import org.graalvm.compiler.phases.util.Providers;
import jdk.vm.ci.aarch64.AArch64;
import jdk.vm.ci.aarch64.AArch64Kind;
import jdk.vm.ci.code.CallingConvention;
import jdk.vm.ci.code.RegisterValue;
import jdk.vm.ci.meta.AllocatableValue;
import jdk.vm.ci.meta.JavaConstant;
import jdk.vm.ci.meta.JavaKind;
import jdk.vm.ci.meta.PlatformKind;
import jdk.vm.ci.meta.PrimitiveConstant;
import jdk.vm.ci.meta.Value;
import jdk.vm.ci.meta.ValueKind;
public abstract class AArch64LIRGenerator extends LIRGenerator {
public AArch64LIRGenerator(LIRKindTool lirKindTool, AArch64ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, Providers providers, LIRGenerationResult lirGenRes) {
super(lirKindTool, arithmeticLIRGen, moveFactory, providers, lirGenRes);
}
protected static final boolean canStoreConstant(JavaConstant c) {
return true;
}
public AllocatableValue moveSp(AllocatableValue val) {
if (val instanceof RegisterValue && ((RegisterValue) val).getRegister().equals(sp)) {
assert val.getPlatformKind() == AArch64Kind.QWORD : "Stackpointer must be long";
return emitMove(val);
}
return val;
}
@Override
public <K extends ValueKind<K>> K toRegisterKind(K kind) {
switch ((AArch64Kind) kind.getPlatformKind()) {
case BYTE:
case WORD:
return kind.changeType(AArch64Kind.DWORD);
default:
return kind;
}
}
@Override
public void emitNullCheck(Value address, LIRFrameState state) {
append(new AArch64Move.NullCheckOp(asAddressValue(address), state));
}
@Override
public Variable emitAddress(AllocatableValue stackslot) {
Variable result = newVariable(LIRKind.value(target().arch.getWordKind()));
append(new AArch64Move.StackLoadAddressOp(result, stackslot));
return result;
}
public AArch64AddressValue asAddressValue(Value address) {
if (address instanceof AArch64AddressValue) {
return (AArch64AddressValue) address;
} else {
return new AArch64AddressValue(address.getValueKind(), asAllocatable(address), Value.ILLEGAL, 0, 1, AddressingMode.BASE_REGISTER_ONLY);
}
}
@Override
public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
Variable prevValue = newVariable(expectedValue.getValueKind());
Variable scratch = newVariable(LIRKind.value(AArch64Kind.DWORD));
append(new CompareAndSwapOp(prevValue, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
assert trueValue.getValueKind().equals(falseValue.getValueKind());
Variable result = newVariable(trueValue.getValueKind());
append(new CondMoveOp(result, ConditionFlag.EQ, asAllocatable(trueValue), asAllocatable(falseValue)));
return result;
}
@Override
public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
Variable result = newVariable(newValue.getValueKind());
Variable scratch = newVariable(LIRKind.value(AArch64Kind.WORD));
append(new CompareAndSwapOp(result, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
return result;
}
@Override
public Value emitAtomicReadAndWrite(Value address, ValueKind<?> kind, Value newValue) {
Variable result = newVariable(kind);
Variable scratch = newVariable(kind);
append(new AtomicReadAndWriteOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(newValue), asAllocatable(scratch)));
return result;
}
@Override
public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) {
Variable result = newVariable(kind);
if (AArch64LIRFlagsVersioned.useLSE(target().arch)) {
append(new AtomicReadAndAddLSEOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(delta)));
} else {
append(new AtomicReadAndAddOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), delta));
}
return result;
}
@Override
public void emitMembar(int barriers) {
int necessaryBarriers = target().arch.requiredBarriers(barriers);
if (target().isMP && necessaryBarriers != 0) {
append(new MembarOp(necessaryBarriers));
}
}
@Override
public void emitJump(LabelRef label) {
assert label != null;
append(new StandardOp.JumpOp(label));
}
@Override
public void emitOverflowCheckBranch(LabelRef overflow, LabelRef noOverflow, LIRKind cmpKind, double overflowProbability) {
append(new AArch64ControlFlow.BranchOp(ConditionFlag.VS, overflow, noOverflow, overflowProbability));
}
@Override
public void emitIntegerTestBranch(Value left, Value right, LabelRef trueDestination, LabelRef falseDestination, double trueSuccessorProbability) {
assert ((AArch64Kind) left.getPlatformKind()).isInteger() && left.getPlatformKind() == right.getPlatformKind();
((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(LIRKind.combine(left, right), AArch64ArithmeticOp.ANDS, true, left, right);
append(new AArch64ControlFlow.BranchOp(ConditionFlag.EQ, trueDestination, falseDestination, trueSuccessorProbability));
}
@Override
public Variable emitConditionalMove(PlatformKind cmpKind, Value left, final Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) {
AArch64ArithmeticLIRGenerator arithLir = ((AArch64ArithmeticLIRGenerator) arithmeticLIRGen);
Value actualRight = right;
if (isJavaConstant(actualRight) && arithLir.mustReplaceNullWithNullRegister((asJavaConstant(actualRight)))) {
actualRight = arithLir.getNullRegisterValue();
}
boolean mirrored = emitCompare(cmpKind, left, actualRight, cond, unorderedIsTrue);
Condition finalCondition = mirrored ? cond.mirror() : cond;
boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
Variable result = newVariable(trueValue.getValueKind());
if (isIntConstant(trueValue, 1) && isIntConstant(falseValue, 0)) {
append(new CondSetOp(result, cmpCondition));
} else if (isIntConstant(trueValue, 0) && isIntConstant(falseValue, 1)) {
append(new CondSetOp(result, cmpCondition.negate()));
} else {
append(new CondMoveOp(result, cmpCondition, loadReg(trueValue), loadReg(falseValue)));
}
return result;
}
@Override
public void emitCompareBranch(PlatformKind cmpKind, Value left, final Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination,
double trueDestinationProbability) {
Value actualRight = right;
if (cond == Condition.EQ) {
assert !LIRValueUtil.isNullConstant(left) : "emitNullCheckBranch()'s null input should be in right.";
AArch64ArithmeticLIRGenerator arithLir = ((AArch64ArithmeticLIRGenerator) arithmeticLIRGen);
if (LIRValueUtil.isNullConstant(actualRight)) {
JavaConstant rightConstant = asJavaConstant(actualRight);
if (arithLir.mustReplaceNullWithNullRegister(rightConstant)) {
actualRight = arithLir.getNullRegisterValue();
} else {
append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination,
trueDestinationProbability));
return;
}
}
AArch64Kind kind = (AArch64Kind) cmpKind;
if (kind.isInteger()) {
if (isIntConstant(left, 0)) {
append(new CompareBranchZeroOp(asAllocatable(actualRight), trueDestination, falseDestination, trueDestinationProbability));
return;
} else if (isIntConstant(actualRight, 0)) {
append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability));
return;
}
}
}
boolean mirrored = emitCompare(cmpKind, left, actualRight, cond, unorderedIsTrue);
Condition finalCondition = mirrored ? cond.mirror() : cond;
boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
append(new BranchOp(cmpCondition, trueDestination, falseDestination, trueDestinationProbability));
}
private static ConditionFlag toConditionFlag(boolean isInt, Condition cond, boolean unorderedIsTrue) {
return isInt ? toIntConditionFlag(cond) : toFloatConditionFlag(cond, unorderedIsTrue);
}
private static ConditionFlag toFloatConditionFlag(Condition cond, boolean unorderedIsTrue) {
switch (cond) {
case LT:
return unorderedIsTrue ? ConditionFlag.LT : ConditionFlag.LO;
case LE:
return unorderedIsTrue ? ConditionFlag.LE : ConditionFlag.LS;
case GE:
return unorderedIsTrue ? ConditionFlag.PL : ConditionFlag.GE;
case GT:
return unorderedIsTrue ? ConditionFlag.HI : ConditionFlag.GT;
case EQ:
return ConditionFlag.EQ;
case NE:
return ConditionFlag.NE;
default:
throw GraalError.shouldNotReachHere();
}
}
private static ConditionFlag toIntConditionFlag(Condition cond) {
switch (cond) {
case EQ:
return ConditionFlag.EQ;
case NE:
return ConditionFlag.NE;
case LT:
return ConditionFlag.LT;
case LE:
return ConditionFlag.LE;
case GT:
return ConditionFlag.GT;
case GE:
return ConditionFlag.GE;
case AE:
return ConditionFlag.HS;
case BE:
return ConditionFlag.LS;
case AT:
return ConditionFlag.HI;
case BT:
return ConditionFlag.LO;
default:
throw GraalError.shouldNotReachHere();
}
}
protected boolean emitCompare(PlatformKind cmpKind, Value a, Value b, Condition condition, boolean unorderedIsTrue) {
Value left;
Value right;
boolean mirrored;
AArch64Kind kind = (AArch64Kind) cmpKind;
if (kind.isInteger()) {
Value aExt = a;
Value bExt = b;
int compareBytes = cmpKind.getSizeInBytes();
if (compareBytes < a.getPlatformKind().getSizeInBytes()) {
aExt = arithmeticLIRGen.emitSignExtend(a, compareBytes * 8, 64);
}
if (compareBytes < b.getPlatformKind().getSizeInBytes()) {
bExt = arithmeticLIRGen.emitSignExtend(b, compareBytes * 8, 64);
}
if (LIRValueUtil.isVariable(bExt)) {
left = load(bExt);
right = loadNonConst(aExt);
mirrored = true;
} else {
left = load(aExt);
right = loadNonConst(bExt);
mirrored = false;
}
append(new AArch64Compare.CompareOp(left, loadNonCompareConst(right)));
} else if (kind.isSIMD()) {
if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(a, condition, unorderedIsTrue)) {
left = load(b);
right = a;
mirrored = true;
} else if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(b, condition, unorderedIsTrue)) {
left = load(a);
right = b;
mirrored = false;
} else {
left = load(a);
right = loadReg(b);
mirrored = false;
}
append(new AArch64Compare.FloatCompareOp(left, asAllocatable(right), condition, unorderedIsTrue));
} else {
throw GraalError.shouldNotReachHere();
}
return mirrored;
}
protected Value loadNonCompareConst(Value value) {
if (!isCompareConstant(value)) {
return loadReg(value);
}
return value;
}
public boolean isCompareConstant(Value value) {
if (isJavaConstant(value)) {
JavaConstant constant = asJavaConstant(value);
if (constant instanceof PrimitiveConstant) {
final long longValue = constant.asLong();
long maskedValue;
switch (constant.getJavaKind()) {
case Boolean:
case Byte:
maskedValue = longValue & 0xFF;
break;
case Char:
case Short:
maskedValue = longValue & 0xFFFF;
break;
case Int:
maskedValue = longValue & 0xFFFF_FFFF;
break;
case Long:
maskedValue = longValue;
break;
default:
throw GraalError.shouldNotReachHere();
}
return AArch64MacroAssembler.isArithmeticImmediate(maskedValue);
} else {
return constant.isDefaultForKind();
}
}
return false;
}
@Override
public Variable emitIntegerTestMove(Value left, Value right, Value trueValue, Value falseValue) {
assert ((AArch64Kind) left.getPlatformKind()).isInteger() && ((AArch64Kind) right.getPlatformKind()).isInteger();
assert ((AArch64Kind) trueValue.getPlatformKind()).isInteger() && ((AArch64Kind) falseValue.getPlatformKind()).isInteger();
((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(left.getValueKind(), AArch64ArithmeticOp.ANDS, true, left, right);
Variable result = newVariable(trueValue.getValueKind());
if (isIntConstant(trueValue, 1) && isIntConstant(falseValue, 0)) {
append(new CondSetOp(result, ConditionFlag.EQ));
} else if (isIntConstant(trueValue, 0) && isIntConstant(falseValue, 1)) {
append(new CondSetOp(result, ConditionFlag.NE));
} else {
append(new CondMoveOp(result, ConditionFlag.EQ, load(trueValue), load(falseValue)));
}
return result;
}
@Override
public void emitStrategySwitch(SwitchStrategy strategy, Variable key, LabelRef[] keyTargets, LabelRef defaultTarget) {
append(createStrategySwitchOp(strategy, keyTargets, defaultTarget, key, newVariable(key.getValueKind()), AArch64LIRGenerator::toIntConditionFlag));
}
protected StrategySwitchOp createStrategySwitchOp(SwitchStrategy strategy, LabelRef[] keyTargets, LabelRef defaultTarget, Variable key, AllocatableValue scratchValue,
Function<Condition, ConditionFlag> converter) {
return new StrategySwitchOp(strategy, keyTargets, defaultTarget, key, scratchValue, converter);
}
@Override
protected void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key) {
append(new TableSwitchOp(lowKey, defaultTarget, targets, key, newVariable(LIRKind.value(target().arch.getWordKind())), newVariable(key.getValueKind())));
}
@Override
public Variable emitByteSwap(Value input) {
Variable result = newVariable(LIRKind.combine(input));
append(new AArch64ByteSwapOp(result, input));
return result;
}
@Override
public Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length1, Value length2) {
LIRKind resultKind = LIRKind.value(AArch64Kind.DWORD);
RegisterValue res = AArch64.r0.asValue(resultKind);
RegisterValue cnt1 = AArch64.r1.asValue(length1.getValueKind());
RegisterValue cnt2 = AArch64.r2.asValue(length2.getValueKind());
emitMove(cnt1, length1);
emitMove(cnt2, length2);
append(new AArch64ArrayCompareToOp(this, kind1, kind2, res, array1, array2, cnt1, cnt2));
Variable result = newVariable(resultKind);
emitMove(result, res);
return result;
}
@Override
public Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, boolean directPointers) {
Variable result = newVariable(LIRKind.value(AArch64Kind.DWORD));
append(new AArch64ArrayEqualsOp(this, kind, result, array1, array2, asAllocatable(length), directPointers));
return result;
}
@Override
protected JavaConstant zapValueForKind(PlatformKind kind) {
long dead = 0xDEADDEADDEADDEADL;
switch ((AArch64Kind) kind) {
case BYTE:
return JavaConstant.forByte((byte) dead);
case WORD:
return JavaConstant.forShort((short) dead);
case DWORD:
return JavaConstant.forInt((int) dead);
case QWORD:
return JavaConstant.forLong(dead);
case SINGLE:
return JavaConstant.forFloat(Float.intBitsToFloat((int) dead));
case DOUBLE:
return JavaConstant.forDouble(Double.longBitsToDouble(dead));
default:
throw GraalError.shouldNotReachHere();
}
}
protected AllocatableValue loadReg(Value val) {
if (!(val instanceof Variable || val instanceof RegisterValue)) {
return emitMove(val);
}
return (AllocatableValue) val;
}
@Override
public void emitPause() {
append(new AArch64PauseOp());
}
public abstract void emitCCall(long address, CallingConvention nativeCallingConvention, Value[] args);
@Override
public void emitSpeculationFence() {
append(new AArch64SpeculativeBarrier());
}
@Override
public void emitZeroMemory(Value address, Value length, boolean isAligned) {
emitZeroMemory(address, length, isAligned, false, -1);
}
protected final void emitZeroMemory(Value address, Value length, boolean isAligned, boolean useDcZva, int zvaLength) {
RegisterValue regAddress = AArch64.r0.asValue(address.getValueKind());
RegisterValue regLength = AArch64.r1.asValue(length.getValueKind());
emitMove(regAddress, address);
emitMove(regLength, length);
append(new AArch64ZeroMemoryOp(regAddress, regLength, isAligned, useDcZva, zvaLength));
}
}