package org.graalvm.compiler.asm.amd64;
import static jdk.vm.ci.amd64.AMD64.rax;
import static jdk.vm.ci.amd64.AMD64.rcx;
import static jdk.vm.ci.amd64.AMD64.rdx;
import static jdk.vm.ci.amd64.AMD64.rsp;
import static org.graalvm.compiler.asm.amd64.AMD64AsmOptions.UseIncDec;
import static org.graalvm.compiler.asm.amd64.AMD64AsmOptions.UseXmmLoadAndClearUpper;
import static org.graalvm.compiler.asm.amd64.AMD64AsmOptions.UseXmmRegToRegMoveAll;
import org.graalvm.compiler.asm.Label;
import org.graalvm.compiler.asm.amd64.AMD64Address.Scale;
import org.graalvm.compiler.core.common.NumUtil;
import jdk.vm.ci.amd64.AMD64;
import jdk.vm.ci.amd64.AMD64Kind;
import jdk.vm.ci.code.Register;
import jdk.vm.ci.code.TargetDescription;
public class AMD64MacroAssembler extends AMD64Assembler {
public AMD64MacroAssembler(TargetDescription target) {
super(target);
}
public final void decrementq(Register reg, int value) {
if (value == Integer.MIN_VALUE) {
subq(reg, value);
return;
}
if (value < 0) {
incrementq(reg, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
decq(reg);
} else {
subq(reg, value);
}
}
public final void decrementq(AMD64Address dst, int value) {
if (value == Integer.MIN_VALUE) {
subq(dst, value);
return;
}
if (value < 0) {
incrementq(dst, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
decq(dst);
} else {
subq(dst, value);
}
}
public void incrementq(Register reg, int value) {
if (value == Integer.MIN_VALUE) {
addq(reg, value);
return;
}
if (value < 0) {
decrementq(reg, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
incq(reg);
} else {
addq(reg, value);
}
}
public final void incrementq(AMD64Address dst, int value) {
if (value == Integer.MIN_VALUE) {
addq(dst, value);
return;
}
if (value < 0) {
decrementq(dst, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
incq(dst);
} else {
addq(dst, value);
}
}
public final void movptr(Register dst, AMD64Address src) {
movq(dst, src);
}
public final void movptr(AMD64Address dst, Register src) {
movq(dst, src);
}
public final void movptr(AMD64Address dst, int src) {
movslq(dst, src);
}
public final void cmpptr(Register src1, Register src2) {
cmpq(src1, src2);
}
public final void cmpptr(Register src1, AMD64Address src2) {
cmpq(src1, src2);
}
public final void decrementl(Register reg) {
decrementl(reg, 1);
}
public final void decrementl(Register reg, int value) {
if (value == Integer.MIN_VALUE) {
subl(reg, value);
return;
}
if (value < 0) {
incrementl(reg, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
decl(reg);
} else {
subl(reg, value);
}
}
public final void decrementl(AMD64Address dst, int value) {
if (value == Integer.MIN_VALUE) {
subl(dst, value);
return;
}
if (value < 0) {
incrementl(dst, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
decl(dst);
} else {
subl(dst, value);
}
}
public final void incrementl(Register reg, int value) {
if (value == Integer.MIN_VALUE) {
addl(reg, value);
return;
}
if (value < 0) {
decrementl(reg, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
incl(reg);
} else {
addl(reg, value);
}
}
public final void incrementl(AMD64Address dst, int value) {
if (value == Integer.MIN_VALUE) {
addl(dst, value);
return;
}
if (value < 0) {
decrementl(dst, -value);
return;
}
if (value == 0) {
return;
}
if (value == 1 && UseIncDec) {
incl(dst);
} else {
addl(dst, value);
}
}
public void movflt(Register dst, Register src) {
assert dst.getRegisterCategory().equals(AMD64.XMM) && src.getRegisterCategory().equals(AMD64.XMM);
if (UseXmmRegToRegMoveAll) {
movaps(dst, src);
} else {
movss(dst, src);
}
}
public void movflt(Register dst, AMD64Address src) {
assert dst.getRegisterCategory().equals(AMD64.XMM);
movss(dst, src);
}
public void movflt(AMD64Address dst, Register src) {
assert src.getRegisterCategory().equals(AMD64.XMM);
movss(dst, src);
}
public void movdbl(Register dst, Register src) {
assert dst.getRegisterCategory().equals(AMD64.XMM) && src.getRegisterCategory().equals(AMD64.XMM);
if (UseXmmRegToRegMoveAll) {
movapd(dst, src);
} else {
movsd(dst, src);
}
}
public void movdbl(Register dst, AMD64Address src) {
assert dst.getRegisterCategory().equals(AMD64.XMM);
if (UseXmmLoadAndClearUpper) {
movsd(dst, src);
} else {
movlpd(dst, src);
}
}
public void movdbl(AMD64Address dst, Register src) {
assert src.getRegisterCategory().equals(AMD64.XMM);
movsd(dst, src);
}
public final void movlong(AMD64Address dst, long src) {
if (NumUtil.isInt(src)) {
AMD64MIOp.MOV.emit(this, OperandSize.QWORD, dst, (int) src);
} else {
AMD64Address high = new AMD64Address(dst.getBase(), dst.getIndex(), dst.getScale(), dst.getDisplacement() + 4);
movl(dst, (int) (src & 0xFFFFFFFF));
movl(high, (int) (src >> 32));
}
}
public final void setl(ConditionFlag cc, Register dst) {
setb(cc, dst);
movzbl(dst, dst);
}
public final void setq(ConditionFlag cc, Register dst) {
setb(cc, dst);
movzbq(dst, dst);
}
public final void flog(Register dest, Register value, boolean base10) {
if (base10) {
fldlg2();
} else {
fldln2();
}
AMD64Address tmp = trigPrologue(value);
fyl2x();
trigEpilogue(dest, tmp);
}
public final void fsin(Register dest, Register value) {
AMD64Address tmp = trigPrologue(value);
fsin();
trigEpilogue(dest, tmp);
}
public final void fcos(Register dest, Register value) {
AMD64Address tmp = trigPrologue(value);
fcos();
trigEpilogue(dest, tmp);
}
public final void ftan(Register dest, Register value) {
AMD64Address tmp = trigPrologue(value);
fptan();
fstp(0);
trigEpilogue(dest, tmp);
}
public final void fpop() {
ffree(0);
fincstp();
}
private AMD64Address trigPrologue(Register value) {
assert value.getRegisterCategory().equals(AMD64.XMM);
AMD64Address tmp = new AMD64Address(AMD64.rsp);
subq(AMD64.rsp, AMD64Kind.DOUBLE.getSizeInBytes());
movdbl(tmp, value);
fldd(tmp);
return tmp;
}
private void trigEpilogue(Register dest, AMD64Address tmp) {
assert dest.getRegisterCategory().equals(AMD64.XMM);
fstpd(tmp);
movdbl(dest, tmp);
addq(AMD64.rsp, AMD64Kind.DOUBLE.getSizeInBytes());
}
public void stringIndexofC8(Register str1, Register str2,
Register cnt1, Register cnt2,
int intCnt2, Register result,
Register vec, Register tmp) {
assert cnt1.equals(rdx) && cnt2.equals(rax) && tmp.equals(rcx) : "pcmpestri";
Label reloadSubstr = new Label();
Label scanToSubstr = new Label();
Label scanSubstr = new Label();
Label retFound = new Label();
Label retNotFound = new Label();
Label exit = new Label();
Label foundSubstr = new Label();
Label matchSubstrHead = new Label();
Label reloadStr = new Label();
Label foundCandidate = new Label();
assert intCnt2 >= 8 : "this code isused only for cnt2 >= 8 chars";
movdqu(vec, new AMD64Address(str2, 0));
movl(cnt2, intCnt2);
movq(result, str1);
if (intCnt2 > 8) {
jmpb(scanToSubstr);
bind(reloadSubstr);
movdqu(vec, new AMD64Address(str2, 0));
negq(cnt2);
bind(reloadStr);
subl(cnt1, cnt2);
addl(cnt1, intCnt2);
movl(cnt2, intCnt2);
decrementl(cnt1, 1);
cmpl(cnt1, cnt2);
jccb(ConditionFlag.Negative, retNotFound);
addq(result, 2);
}
bind(scanToSubstr);
pcmpestri(vec, new AMD64Address(result, 0), 0x0d);
jccb(ConditionFlag.Below, foundCandidate);
subl(cnt1, 8);
jccb(ConditionFlag.LessEqual, retNotFound);
cmpl(cnt1, cnt2);
jccb(ConditionFlag.Negative, retNotFound);
addq(result, 16);
jmpb(scanToSubstr);
bind(foundCandidate);
if (intCnt2 == 8) {
jccb(ConditionFlag.Overflow, retFound);
} else {
jccb(ConditionFlag.Overflow, foundSubstr);
}
leaq(result, new AMD64Address(result, tmp, Scale.Times2, 0));
subl(cnt1, tmp);
cmpl(cnt1, cnt2);
if (intCnt2 == 8) {
jccb(ConditionFlag.GreaterEqual, scanToSubstr);
} else {
jccb(ConditionFlag.GreaterEqual, matchSubstrHead);
}
bind(retNotFound);
movl(result, -1);
jmpb(exit);
if (intCnt2 > 8) {
bind(matchSubstrHead);
pcmpestri(vec, new AMD64Address(result, 0), 0x0d);
jccb(ConditionFlag.NoOverflow, reloadStr);
Label contScanSubstr = new Label();
bind(foundSubstr);
negq(cnt2);
addq(cnt2, 8);
bind(scanSubstr);
subl(cnt1, 8);
cmpl(cnt2, -8);
jccb(ConditionFlag.LessEqual, contScanSubstr);
addl(cnt1, cnt2);
addl(cnt1, 8);
movl(cnt2, 8);
negq(cnt2);
bind(contScanSubstr);
if (intCnt2 < 1024 * 1024 * 1024) {
movdqu(vec, new AMD64Address(str2, cnt2, Scale.Times2, intCnt2 * 2));
pcmpestri(vec, new AMD64Address(result, cnt2, Scale.Times2, intCnt2 * 2), 0x0d);
} else {
movl(tmp, intCnt2);
addq(tmp, cnt2);
movdqu(vec, new AMD64Address(str2, tmp, Scale.Times2, 0));
pcmpestri(vec, new AMD64Address(result, tmp, Scale.Times2, 0), 0x0d);
}
jcc(ConditionFlag.NoOverflow, reloadSubstr);
addq(cnt2, 8);
jcc(ConditionFlag.Negative, scanSubstr);
}
bind(retFound);
subq(result, str1);
shrl(result, 1);
bind(exit);
}
public void stringIndexOf(Register str1, Register str2,
Register cnt1, Register cnt2,
int intCnt2, Register result,
Register vec, Register tmp, int vmPageSize) {
assert intCnt2 == -1 || (0 < intCnt2 && intCnt2 < 8) : "should be != 0";
assert cnt1.equals(rdx) && cnt2.equals(rax) && tmp.equals(rcx) : "pcmpestri";
Label reloadSubstr = new Label();
Label scanToSubstr = new Label();
Label scanSubstr = new Label();
Label adjustStr = new Label();
Label retFound = new Label();
Label retNotFound = new Label();
Label cleanup = new Label();
Label foundSubstr = new Label();
Label foundCandidate = new Label();
int wordSize = 8;
Label bigStrings = new Label();
Label checkStr = new Label();
Label copySubstr = new Label();
Label copyStr = new Label();
movq(tmp, rsp);
if (intCnt2 > 0) {
if (intCnt2 == 1) {
movzwl(result, new AMD64Address(str2, 0));
movdl(vec, result);
} else if (intCnt2 == 2) {
movdl(vec, new AMD64Address(str2, 0));
} else if (intCnt2 == 4) {
movq(vec, new AMD64Address(str2, 0));
} else {
movdqu(vec, new AMD64Address(str2, (intCnt2 * 2) - 16));
psrldq(vec, 16 - (intCnt2 * 2));
}
} else {
cmpl(cnt2, 8);
jccb(ConditionFlag.AboveEqual, bigStrings);
assert vmPageSize < 1024 * 1024 * 1024 : "default page should be small";
movl(result, str2);
andl(result, (vmPageSize - 1));
cmpl(result, (vmPageSize - 16));
jccb(ConditionFlag.BelowEqual, checkStr);
subq(rsp, 16);
int stackOffset = wordSize - 2;
push(cnt2);
bind(copySubstr);
movzwl(result, new AMD64Address(str2, cnt2, Scale.Times2, -2));
movw(new AMD64Address(rsp, cnt2, Scale.Times2, stackOffset), result);
decrementl(cnt2, 1);
jccb(ConditionFlag.NotZero, copySubstr);
pop(cnt2);
movq(str2, rsp);
}
bind(checkStr);
cmpl(cnt1, 8);
jccb(ConditionFlag.AboveEqual, bigStrings);
movl(result, str1);
andl(result, (vmPageSize - 1));
cmpl(result, (vmPageSize - 16));
jccb(ConditionFlag.BelowEqual, bigStrings);
subq(rsp, 16);
int stackOffset = -2;
if (intCnt2 < 0) {
push(cnt2);
stackOffset += wordSize;
}
movl(cnt2, cnt1);
bind(copyStr);
movzwl(result, new AMD64Address(str1, cnt2, Scale.Times2, -2));
movw(new AMD64Address(rsp, cnt2, Scale.Times2, stackOffset), result);
decrementl(cnt2, 1);
jccb(ConditionFlag.NotZero, copyStr);
if (intCnt2 < 0) {
pop(cnt2);
}
movq(str1, rsp);
bind(bigStrings);
if (intCnt2 < 0) {
movdqu(vec, new AMD64Address(str2, 0));
push(cnt2);
push(str2);
push(str1);
} else {
movl(cnt2, intCnt2);
}
push(tmp);
movq(result, str1);
if (intCnt2 < 0) {
jmpb(scanToSubstr);
bind(reloadSubstr);
movq(str2, new AMD64Address(rsp, 2 * wordSize));
movl(cnt2, new AMD64Address(rsp, 3 * wordSize));
movdqu(vec, new AMD64Address(str2, 0));
subq(str1, result);
shrl(str1, 1);
addl(cnt1, str1);
decrementl(cnt1);
cmpl(cnt1, cnt2);
jccb(ConditionFlag.Negative, retNotFound);
addq(result, 2);
}
bind(scanToSubstr);
assert cnt1.equals(rdx) && cnt2.equals(rax) && tmp.equals(rcx) : "pcmpestri";
pcmpestri(vec, new AMD64Address(result, 0), 0x0d);
jccb(ConditionFlag.Below, foundCandidate);
subl(cnt1, 8);
jccb(ConditionFlag.LessEqual, retNotFound);
cmpl(cnt1, cnt2);
jccb(ConditionFlag.Negative, retNotFound);
addq(result, 16);
bind(adjustStr);
cmpl(cnt1, 8);
jccb(ConditionFlag.GreaterEqual, scanToSubstr);
leaq(result, new AMD64Address(result, cnt1, Scale.Times2, -16));
movl(cnt1, 8);
jmpb(scanToSubstr);
bind(foundCandidate);
subl(cnt1, tmp);
cmpl(cnt1, cnt2);
jccb(ConditionFlag.GreaterEqual, foundSubstr);
bind(retNotFound);
movl(result, -1);
jmpb(cleanup);
bind(foundSubstr);
leaq(result, new AMD64Address(result, tmp, Scale.Times2));
if (intCnt2 > 0) {
cmpl(tmp, 8 - intCnt2);
jccb(ConditionFlag.Greater, adjustStr);
} else {
assert intCnt2 == -1 : "should be != 0";
addl(tmp, cnt2);
cmpl(tmp, 8);
jccb(ConditionFlag.LessEqual, retFound);
cmpl(cnt2, 8);
jccb(ConditionFlag.LessEqual, adjustStr);
Label checkNext = new Label();
Label contScanSubstr = new Label();
Label retFoundLong = new Label();
movq(str1, result);
cmpl(tmp, cnt2);
jccb(ConditionFlag.Equal, checkNext);
bind(scanSubstr);
pcmpestri(vec, new AMD64Address(str1, 0), 0x0d);
jcc(ConditionFlag.NoOverflow, reloadSubstr);
bind(checkNext);
subl(cnt2, 8);
jccb(ConditionFlag.LessEqual, retFoundLong);
addq(str1, 16);
addq(str2, 16);
subl(cnt1, 8);
cmpl(cnt2, 8);
jccb(ConditionFlag.GreaterEqual, contScanSubstr);
leaq(str2, new AMD64Address(str2, cnt2, Scale.Times2, -16));
leaq(str1, new AMD64Address(str1, cnt2, Scale.Times2, -16));
subl(cnt1, cnt2);
movl(cnt2, 8);
addl(cnt1, 8);
bind(contScanSubstr);
movdqu(vec, new AMD64Address(str2, 0));
jmpb(scanSubstr);
bind(retFoundLong);
movq(str1, new AMD64Address(rsp, wordSize));
}
bind(retFound);
subq(result, str1);
shrl(result, 1);
bind(cleanup);
pop(rsp);
}
}