package org.graalvm.compiler.core.aarch64;
import static jdk.vm.ci.aarch64.AArch64Kind.DWORD;
import static jdk.vm.ci.aarch64.AArch64Kind.QWORD;
import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant;
import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.BSR;
import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.CLZ;
import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.CTZ;
import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.POPCNT;
import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
import org.graalvm.compiler.core.common.LIRKind;
import org.graalvm.compiler.core.common.NumUtil;
import org.graalvm.compiler.core.common.calc.FloatConvert;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.lir.ConstantValue;
import org.graalvm.compiler.lir.LIRFrameState;
import org.graalvm.compiler.lir.Variable;
import org.graalvm.compiler.lir.aarch64.AArch64AddressValue;
import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticLIRGeneratorTool;
import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp;
import org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp;
import org.graalvm.compiler.lir.aarch64.AArch64Move;
import org.graalvm.compiler.lir.aarch64.AArch64Move.LoadOp;
import org.graalvm.compiler.lir.aarch64.AArch64Move.StoreConstantOp;
import org.graalvm.compiler.lir.aarch64.AArch64Move.StoreOp;
import org.graalvm.compiler.lir.aarch64.AArch64ReinterpretOp;
import org.graalvm.compiler.lir.aarch64.AArch64SignExtendOp;
import org.graalvm.compiler.lir.aarch64.AArch64Unary;
import org.graalvm.compiler.lir.gen.ArithmeticLIRGenerator;
import jdk.vm.ci.aarch64.AArch64Kind;
import jdk.vm.ci.meta.AllocatableValue;
import jdk.vm.ci.meta.JavaConstant;
import jdk.vm.ci.meta.PlatformKind;
import jdk.vm.ci.meta.Value;
import jdk.vm.ci.meta.ValueKind;
public class AArch64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implements AArch64ArithmeticLIRGeneratorTool {
public AArch64ArithmeticLIRGenerator(AllocatableValue nullRegisterValue) {
this.nullRegisterValue = nullRegisterValue;
}
private final AllocatableValue nullRegisterValue;
@Override
public AArch64LIRGenerator getLIRGen() {
return (AArch64LIRGenerator) super.getLIRGen();
}
public boolean mustReplaceNullWithNullRegister(JavaConstant nullConstant) {
return nullRegisterValue != null && JavaConstant.NULL_POINTER.equals(nullConstant);
}
public AllocatableValue getNullRegisterValue() {
return nullRegisterValue;
}
@Override
protected boolean isNumericInteger(PlatformKind kind) {
return ((AArch64Kind) kind).isInteger();
}
@Override
protected Variable emitAdd(LIRKind resultKind, Value a, Value b, boolean setFlags) {
if (isNumericInteger(a.getPlatformKind())) {
AArch64ArithmeticOp op = setFlags ? AArch64ArithmeticOp.ADDS : AArch64ArithmeticOp.ADD;
return emitBinary(resultKind, op, true, a, b);
} else {
assert !setFlags : "Cannot set flags on floating point arithmetic";
return emitBinary(resultKind, AArch64ArithmeticOp.FADD, true, a, b);
}
}
@Override
protected Variable emitSub(LIRKind resultKind, Value a, Value b, boolean setFlags) {
if (isNumericInteger(a.getPlatformKind())) {
AArch64ArithmeticOp op = setFlags ? AArch64ArithmeticOp.SUBS : AArch64ArithmeticOp.SUB;
return emitBinary(resultKind, op, false, a, b);
} else {
assert !setFlags : "Cannot set flags on floating point arithmetic";
return emitBinary(resultKind, AArch64ArithmeticOp.FSUB, false, a, b);
}
}
public Value emitExtendMemory(boolean isSigned, AArch64Kind memoryKind, int resultBits, AArch64AddressValue address, LIRFrameState state) {
Variable result = getLIRGen().newVariable(LIRKind.value(resultBits == 32 ? AArch64Kind.DWORD : AArch64Kind.QWORD));
int targetSize = resultBits <= 32 ? 32 : 64;
switch (memoryKind) {
case BYTE:
case WORD:
case DWORD:
case QWORD:
getLIRGen().append(new AArch64Unary.MemoryOp(isSigned, targetSize,
memoryKind.getSizeInBytes() * 8, result, address, state));
break;
default:
throw GraalError.shouldNotReachHere();
}
return result;
}
@Override
public Value emitMul(Value a, Value b, boolean setFlags) {
AArch64ArithmeticOp intOp = setFlags ? AArch64ArithmeticOp.MULVS : AArch64ArithmeticOp.MUL;
return emitBinary(LIRKind.combine(a, b), getOpCode(a, intOp, AArch64ArithmeticOp.FMUL), true, a, b);
}
@Override
public Value emitMulHigh(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.SMULH, true, a, b);
}
@Override
public Value emitUMulHigh(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.UMULH, true, a, b);
}
public Value emitMNeg(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind()) && isNumericInteger(b.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.MNEG, true, a, b);
}
@Override
public Value emitDiv(Value a, Value b, LIRFrameState state) {
return emitBinary(LIRKind.combine(a, b), getOpCode(a, AArch64ArithmeticOp.DIV, AArch64ArithmeticOp.FDIV), false, asAllocatable(a), asAllocatable(b));
}
@Override
public Value emitRem(Value a, Value b, LIRFrameState state) {
return emitBinary(LIRKind.combine(a, b), getOpCode(a, AArch64ArithmeticOp.REM, AArch64ArithmeticOp.FREM), false, asAllocatable(a), asAllocatable(b));
}
@Override
public Value emitUDiv(Value a, Value b, LIRFrameState state) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.UDIV, false, asAllocatable(a), asAllocatable(b));
}
@Override
public Value emitURem(Value a, Value b, LIRFrameState state) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.UREM, false, asAllocatable(a), asAllocatable(b));
}
@Override
public Value emitAnd(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.AND, true, a, b);
}
@Override
public Value emitOr(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.OR, true, a, b);
}
@Override
public Value emitXor(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.XOR, true, a, b);
}
@Override
public Value emitShl(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.SHL, false, a, b);
}
@Override
public Value emitShr(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.ASHR, false, a, b);
}
@Override
public Value emitUShr(Value a, Value b) {
assert isNumericInteger(a.getPlatformKind());
return emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.LSHR, false, a, b);
}
@Override
public Value emitFloatConvert(FloatConvert op, Value inputVal) {
PlatformKind resultPlatformKind = getFloatConvertResultKind(op);
LIRKind resultLirKind = LIRKind.combine(inputVal).changeType(resultPlatformKind);
Variable result = getLIRGen().newVariable(resultLirKind);
getLIRGen().append(new AArch64FloatConvertOp(op, result, asAllocatable(inputVal)));
return result;
}
Value emitIntegerMAdd(Value a, Value b, Value c, boolean isI2L) {
return emitMultiplyAddSub(isI2L ? AArch64ArithmeticOp.SMADDL : AArch64ArithmeticOp.MADD, a, b, c);
}
Value emitIntegerMSub(Value a, Value b, Value c, boolean isI2L) {
return emitMultiplyAddSub(isI2L ? AArch64ArithmeticOp.SMSUBL : AArch64ArithmeticOp.MSUB, a, b, c);
}
private Value emitMultiplyAddSub(AArch64ArithmeticOp op, Value a, Value b, Value c) {
assert a.getPlatformKind() == b.getPlatformKind();
Variable result;
if (op == AArch64ArithmeticOp.SMADDL || op == AArch64ArithmeticOp.SMSUBL) {
assert a.getPlatformKind() != c.getPlatformKind();
result = getLIRGen().newVariable(LIRKind.combine(c));
} else {
assert a.getPlatformKind() == c.getPlatformKind();
if (op == AArch64ArithmeticOp.FMADD) {
assert a.getPlatformKind() == AArch64Kind.SINGLE || a.getPlatformKind() == AArch64Kind.DOUBLE;
} else {
assert op == AArch64ArithmeticOp.MADD || op == AArch64ArithmeticOp.MSUB;
assert isNumericInteger(a.getPlatformKind());
}
result = getLIRGen().newVariable(LIRKind.combine(a, b, c));
}
AllocatableValue x = moveSp(asAllocatable(a));
AllocatableValue y = moveSp(asAllocatable(b));
AllocatableValue z = moveSp(asAllocatable(c));
getLIRGen().append(new AArch64ArithmeticOp.MultiplyAddSubOp(op, result, x, y, z));
return result;
}
private static PlatformKind getFloatConvertResultKind(FloatConvert op) {
switch (op) {
case F2I:
case D2I:
return AArch64Kind.DWORD;
case F2L:
case D2L:
return AArch64Kind.QWORD;
case I2F:
case L2F:
case D2F:
return AArch64Kind.SINGLE;
case I2D:
case L2D:
case F2D:
return AArch64Kind.DOUBLE;
default:
throw GraalError.shouldNotReachHere();
}
}
@Override
public Value emitReinterpret(LIRKind to, Value inputVal) {
ValueKind<?> from = inputVal.getValueKind();
if (to.equals(from)) {
return inputVal;
}
Variable result = getLIRGen().newVariable(to);
getLIRGen().append(new AArch64ReinterpretOp(result, asAllocatable(inputVal)));
return result;
}
@Override
public Value emitNarrow(Value inputVal, int bits) {
if (inputVal.getPlatformKind() == AArch64Kind.QWORD && bits <= 32) {
LIRKind resultKind = getResultLirKind(bits, inputVal);
long mask = NumUtil.getNbitNumberLong(bits);
Value maskValue = new ConstantValue(resultKind, JavaConstant.forLong(mask));
return emitBinary(resultKind, AArch64ArithmeticOp.AND, true, inputVal, maskValue);
} else {
return inputVal;
}
}
@Override
public Value emitZeroExtend(Value inputVal, int fromBits, int toBits) {
assert fromBits <= toBits && toBits <= 64;
if (fromBits == toBits) {
return inputVal;
}
LIRKind resultKind = getResultLirKind(toBits, inputVal);
long mask = NumUtil.getNbitNumberLong(fromBits);
Value maskValue = new ConstantValue(resultKind, JavaConstant.forLong(mask));
return emitBinary(resultKind, AArch64ArithmeticOp.AND, true, inputVal, maskValue);
}
@Override
public Value emitSignExtend(Value inputVal, int fromBits, int toBits) {
LIRKind resultKind = getResultLirKind(toBits, inputVal);
assert fromBits <= toBits && toBits <= 64;
if (fromBits == toBits) {
return inputVal;
} else if (isJavaConstant(inputVal)) {
JavaConstant javaConstant = asJavaConstant(inputVal);
long constant;
if (javaConstant.isNull()) {
constant = 0;
} else {
constant = javaConstant.asLong();
}
int shiftCount = QWORD.getSizeInBytes() * 8 - fromBits;
return new ConstantValue(resultKind, JavaConstant.forLong((constant << shiftCount) >> shiftCount));
}
Variable result = getLIRGen().newVariable(resultKind);
getLIRGen().append(new AArch64SignExtendOp(result, asAllocatable(inputVal), fromBits, toBits));
return result;
}
private static LIRKind getResultLirKind(int resultBitSize, Value... inputValues) {
if (resultBitSize == 64) {
return LIRKind.combine(inputValues).changeType(QWORD);
} else {
return LIRKind.combine(inputValues).changeType(DWORD);
}
}
protected Variable emitBinary(ValueKind<?> resultKind, AArch64ArithmeticOp op, boolean commutative, Value a, Value b) {
Variable result = getLIRGen().newVariable(resultKind);
if (isValidBinaryConstant(op, a, b)) {
emitBinaryConst(result, op, asAllocatable(a), asJavaConstant(b));
} else if (commutative && isValidBinaryConstant(op, b, a)) {
emitBinaryConst(result, op, asAllocatable(b), asJavaConstant(a));
} else {
emitBinaryVar(result, op, asAllocatable(a), asAllocatable(b));
}
return result;
}
private void emitBinaryVar(Variable result, AArch64ArithmeticOp op, AllocatableValue a, AllocatableValue b) {
AllocatableValue x = moveSp(a);
AllocatableValue y = moveSp(b);
switch (op) {
case FREM:
case REM:
case UREM:
getLIRGen().append(new AArch64ArithmeticOp.BinaryCompositeOp(op, result, x, y));
break;
default:
getLIRGen().append(new AArch64ArithmeticOp.BinaryOp(op, result, x, y));
break;
}
}
private void emitBinaryConst(Variable result, AArch64ArithmeticOp op, AllocatableValue a, JavaConstant b) {
AllocatableValue x = moveSp(a);
getLIRGen().append(new AArch64ArithmeticOp.BinaryConstOp(op, result, x, b));
}
private static boolean isValidBinaryConstant(AArch64ArithmeticOp op, Value a, Value b) {
if (!isJavaConstant(b)) {
return false;
}
JavaConstant constValue = asJavaConstant(b);
switch (op.category) {
case LOGICAL:
return isLogicalConstant(constValue);
case ARITHMETIC:
return isArithmeticConstant(constValue);
case SHIFT:
assert constValue.asLong() >= 0 && constValue.asLong() < a.getPlatformKind().getSizeInBytes() * Byte.SIZE;
return true;
case NONE:
return false;
default:
throw GraalError.shouldNotReachHere();
}
}
private static boolean isLogicalConstant(JavaConstant constValue) {
switch (constValue.getJavaKind()) {
case Int:
return AArch64MacroAssembler.isLogicalImmediate(constValue.asInt());
case Long:
return AArch64MacroAssembler.isLogicalImmediate(constValue.asLong());
default:
return false;
}
}
protected static boolean isArithmeticConstant(JavaConstant constValue) {
switch (constValue.getJavaKind()) {
case Int:
case Long:
return AArch64MacroAssembler.isArithmeticImmediate(constValue.asLong());
case Object:
return constValue.isNull();
default:
return false;
}
}
@Override
public Value emitNegate(Value inputVal) {
return emitUnary(getOpCode(inputVal, AArch64ArithmeticOp.NEG, AArch64ArithmeticOp.FNEG), inputVal);
}
@Override
public Value emitNot(Value input) {
assert isNumericInteger(input.getPlatformKind());
return emitUnary(AArch64ArithmeticOp.NOT, input);
}
@Override
public Value emitMathAbs(Value input) {
return emitUnary(getOpCode(input, AArch64ArithmeticOp.ABS, AArch64ArithmeticOp.FABS), input);
}
@Override
public Value emitMathSqrt(Value input) {
assert input.getPlatformKind() == AArch64Kind.DOUBLE ||
input.getPlatformKind() == AArch64Kind.SINGLE;
return emitUnary(AArch64ArithmeticOp.SQRT, input);
}
@Override
public Variable emitBitScanForward(Value value) {
throw GraalError.unimplemented();
}
@Override
public Value emitBitCount(Value operand) {
assert ((AArch64Kind) operand.getPlatformKind()).isInteger();
Variable result = getLIRGen().newVariable(LIRKind.combine(operand).changeType(AArch64Kind.DWORD));
getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), POPCNT, result, asAllocatable(operand)));
return result;
}
@Override
public Value emitBitScanReverse(Value value) {
Variable result = getLIRGen().newVariable(LIRKind.combine(value).changeType(AArch64Kind.DWORD));
getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), BSR, result, asAllocatable(value)));
return result;
}
@Override
public Value emitFusedMultiplyAdd(Value a, Value b, Value c) {
return emitMultiplyAddSub(AArch64ArithmeticOp.FMADD, a, b, c);
}
@Override
public Value emitCountLeadingZeros(Value value) {
Variable result = getLIRGen().newVariable(LIRKind.combine(value).changeType(AArch64Kind.DWORD));
getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), CLZ, result, asAllocatable(value)));
return result;
}
@Override
public Value emitCountTrailingZeros(Value value) {
Variable result = getLIRGen().newVariable(LIRKind.combine(value).changeType(AArch64Kind.DWORD));
getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), CTZ, result, asAllocatable(value)));
return result;
}
private Variable emitUnary(AArch64ArithmeticOp op, Value inputVal) {
AllocatableValue input = asAllocatable(inputVal);
Variable result = getLIRGen().newVariable(LIRKind.combine(input));
getLIRGen().append(new AArch64ArithmeticOp.UnaryOp(op, result, input));
return result;
}
private AllocatableValue moveSp(AllocatableValue val) {
return getLIRGen().moveSp(val);
}
private AArch64ArithmeticOp getOpCode(Value val, AArch64ArithmeticOp intOp, AArch64ArithmeticOp floatOp) {
return isNumericInteger(val.getPlatformKind()) ? intOp : floatOp;
}
@Override
public Variable emitLoad(LIRKind kind, Value address, LIRFrameState state) {
AArch64AddressValue loadAddress = getLIRGen().asAddressValue(address);
Variable result = getLIRGen().newVariable(getLIRGen().toRegisterKind(kind));
getLIRGen().append(new LoadOp((AArch64Kind) kind.getPlatformKind(), result, loadAddress, state));
return result;
}
@Override
public Variable emitVolatileLoad(LIRKind kind, Value address, LIRFrameState state) {
AllocatableValue loadAddress = asAllocatable(address);
Variable result = getLIRGen().newVariable(getLIRGen().toRegisterKind(kind));
getLIRGen().append(new AArch64Move.VolatileLoadOp((AArch64Kind) kind.getPlatformKind(), result, loadAddress, state));
return result;
}
@Override
public void emitStore(ValueKind<?> lirKind, Value address, Value inputVal, LIRFrameState state) {
AArch64AddressValue storeAddress = getLIRGen().asAddressValue(address);
AArch64Kind kind = (AArch64Kind) lirKind.getPlatformKind();
if (isJavaConstant(inputVal) && kind.isInteger()) {
JavaConstant c = asJavaConstant(inputVal);
if (c.isDefaultForKind()) {
getLIRGen().append(new StoreConstantOp(kind, storeAddress, c, state));
return;
}
}
AllocatableValue input = asAllocatable(inputVal);
getLIRGen().append(new StoreOp(kind, storeAddress, input, state));
}
@Override
public void emitVolatileStore(ValueKind<?> lirKind, Value addressVal, Value inputVal, LIRFrameState state) {
AArch64Kind kind = (AArch64Kind) lirKind.getPlatformKind();
AllocatableValue input = asAllocatable(inputVal);
AllocatableValue address = asAllocatable(addressVal);
getLIRGen().append(new AArch64Move.VolatileStoreOp(kind, address, input, state));
}
@Override
public Value emitRound(Value value, RoundingMode mode) {
AArch64ArithmeticOp op;
switch (mode) {
case NEAREST:
op = AArch64ArithmeticOp.FRINTN;
break;
case UP:
op = AArch64ArithmeticOp.FRINTP;
break;
case DOWN:
op = AArch64ArithmeticOp.FRINTM;
break;
default:
throw GraalError.shouldNotReachHere();
}
return emitUnary(op, value);
}
}