Google Git
Sign in
webkit / WebKit / cca3b771b524f4d40cac25de02ae46d5189d1489 / . / Source / JavaScriptCore / assembler / MacroAssemblerX86Common.h
blob: 30cc4e4bcafffdb94cff8046c051e1f2e98870d9 [file] [log] [blame]
/*
* Copyright (C) 2008-2021 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#if ENABLE(ASSEMBLER)
#include "X86Assembler.h"
#include "AbstractMacroAssembler.h"
#include <array>
namespace JSC {
using Assembler = TARGET_ASSEMBLER;
class MacroAssemblerX86Common : public AbstractMacroAssembler<Assembler> {
public:
#if CPU(X86_64)
// Use this directly only if you're not generating code with it.
static constexpr X86Registers::RegisterID s_scratchRegister = X86Registers::r11;
// Use this when generating code so that we get enforcement of the disallowing of scratch register
// usage.
X86Registers::RegisterID scratchRegister()
{
RELEASE_ASSERT(m_allowScratchRegister);
return s_scratchRegister;
}
#endif
protected:
static constexpr int DoubleConditionBitInvert = 0x10;
static constexpr int DoubleConditionBitSpecial = 0x20;
static constexpr int DoubleConditionBits = DoubleConditionBitInvert | DoubleConditionBitSpecial;
public:
typedef X86Assembler::XMMRegisterID XMMRegisterID;
static bool isCompactPtrAlignedAddressOffset(ptrdiff_t value)
{
return value >= -128 && value <= 127;
}
enum RelationalCondition {
Equal = X86Assembler::ConditionE,
NotEqual = X86Assembler::ConditionNE,
Above = X86Assembler::ConditionA,
AboveOrEqual = X86Assembler::ConditionAE,
Below = X86Assembler::ConditionB,
BelowOrEqual = X86Assembler::ConditionBE,
GreaterThan = X86Assembler::ConditionG,
GreaterThanOrEqual = X86Assembler::ConditionGE,
LessThan = X86Assembler::ConditionL,
LessThanOrEqual = X86Assembler::ConditionLE
};
enum ResultCondition {
Overflow = X86Assembler::ConditionO,
Signed = X86Assembler::ConditionS,
PositiveOrZero = X86Assembler::ConditionNS,
Zero = X86Assembler::ConditionE,
NonZero = X86Assembler::ConditionNE
};
// FIXME: it would be neat to rename this to FloatingPointCondition in every assembler.
enum DoubleCondition {
// These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN.
DoubleEqualAndOrdered = X86Assembler::ConditionE | DoubleConditionBitSpecial,
DoubleNotEqualAndOrdered = X86Assembler::ConditionNE,
DoubleGreaterThanAndOrdered = X86Assembler::ConditionA,
DoubleGreaterThanOrEqualAndOrdered = X86Assembler::ConditionAE,
DoubleLessThanAndOrdered = X86Assembler::ConditionA | DoubleConditionBitInvert,
DoubleLessThanOrEqualAndOrdered = X86Assembler::ConditionAE | DoubleConditionBitInvert,
// If either operand is NaN, these conditions always evaluate to true.
DoubleEqualOrUnordered = X86Assembler::ConditionE,
DoubleNotEqualOrUnordered = X86Assembler::ConditionNE | DoubleConditionBitSpecial,
DoubleGreaterThanOrUnordered = X86Assembler::ConditionB | DoubleConditionBitInvert,
DoubleGreaterThanOrEqualOrUnordered = X86Assembler::ConditionBE | DoubleConditionBitInvert,
DoubleLessThanOrUnordered = X86Assembler::ConditionB,
DoubleLessThanOrEqualOrUnordered = X86Assembler::ConditionBE,
};
COMPILE_ASSERT(
!((X86Assembler::ConditionE | X86Assembler::ConditionNE | X86Assembler::ConditionA | X86Assembler::ConditionAE | X86Assembler::ConditionB | X86Assembler::ConditionBE) & DoubleConditionBits),
DoubleConditionBits_should_not_interfere_with_X86Assembler_Condition_codes);
static constexpr RegisterID stackPointerRegister = X86Registers::esp;
static constexpr RegisterID framePointerRegister = X86Registers::ebp;
static bool constexpr canBlind() { return true; }
static bool constexpr shouldBlindForSpecificArch(uint32_t value) { return value >= 0x00ffffff; }
static bool constexpr shouldBlindForSpecificArch(uint64_t value) { return value >= 0x00ffffff; }
// Integer arithmetic operations:
//
// Operations are typically two operand - operation(source, srcDst)
// For many operations the source may be an TrustedImm32, the srcDst operand
// may often be a memory location (explictly described using an Address
// object).
void add32(RegisterID src, RegisterID dest)
{
m_assembler.addl_rr(src, dest);
}
void add32(TrustedImm32 imm, Address address)
{
m_assembler.addl_im(imm.m_value, address.offset, address.base);
}
void add32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.addl_im(imm.m_value, address.offset, address.base, address.index, address.scale);
}
void add8(TrustedImm32 imm, Address address)
{
TrustedImm32 imm8(static_cast<int8_t>(imm.m_value));
m_assembler.addb_im(imm8.m_value, address.offset, address.base);
}
void add8(TrustedImm32 imm, BaseIndex address)
{
TrustedImm32 imm8(static_cast<int8_t>(imm.m_value));
m_assembler.addb_im(imm8.m_value, address.offset, address.base, address.index, address.scale);
}
void add16(TrustedImm32 imm, Address address)
{
m_assembler.addw_im(imm.m_value, address.offset, address.base);
}
void add16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.addw_im(imm.m_value, address.offset, address.base, address.index, address.scale);
}
void add32(TrustedImm32 imm, RegisterID dest)
{
if (imm.m_value == 1)
m_assembler.inc_r(dest);
else
m_assembler.addl_ir(imm.m_value, dest);
}
void add32(Address src, RegisterID dest)
{
m_assembler.addl_mr(src.offset, src.base, dest);
}
void add32(BaseIndex src, RegisterID dest)
{
m_assembler.addl_mr(src.offset, src.base, src.index, src.scale, dest);
}
void add32(RegisterID src, Address dest)
{
m_assembler.addl_rm(src, dest.offset, dest.base);
}
void add32(RegisterID src, BaseIndex dest)
{
m_assembler.addl_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void add8(RegisterID src, Address dest)
{
m_assembler.addb_rm(src, dest.offset, dest.base);
}
void add8(RegisterID src, BaseIndex dest)
{
m_assembler.addb_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void add16(RegisterID src, Address dest)
{
m_assembler.addw_rm(src, dest.offset, dest.base);
}
void add16(RegisterID src, BaseIndex dest)
{
m_assembler.addw_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void add32(TrustedImm32 imm, RegisterID src, RegisterID dest)
{
if (!imm.m_value) {
zeroExtend32ToWord(src, dest);
return;
}
if (src == dest) {
add32(imm, dest);
return;
}
m_assembler.leal_mr(imm.m_value, src, dest);
}
void add32(RegisterID a, RegisterID b, RegisterID dest)
{
x86Lea32(BaseIndex(a, b, TimesOne), dest);
}
void x86Lea32(BaseIndex index, RegisterID dest)
{
if (!index.scale && !index.offset) {
if (index.base == dest) {
add32(index.index, dest);
return;
}
if (index.index == dest) {
add32(index.base, dest);
return;
}
}
m_assembler.leal_mr(index.offset, index.base, index.index, index.scale, dest);
}
void and32(RegisterID src, RegisterID dest)
{
m_assembler.andl_rr(src, dest);
}
void and32(TrustedImm32 imm, RegisterID dest)
{
m_assembler.andl_ir(imm.m_value, dest);
}
void and32(RegisterID src, Address dest)
{
m_assembler.andl_rm(src, dest.offset, dest.base);
}
void and32(RegisterID src, BaseIndex dest)
{
m_assembler.andl_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void and16(RegisterID src, Address dest)
{
m_assembler.andw_rm(src, dest.offset, dest.base);
}
void and16(RegisterID src, BaseIndex dest)
{
m_assembler.andw_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void and8(RegisterID src, Address dest)
{
m_assembler.andb_rm(src, dest.offset, dest.base);
}
void and8(RegisterID src, BaseIndex dest)
{
m_assembler.andb_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void and32(Address src, RegisterID dest)
{
m_assembler.andl_mr(src.offset, src.base, dest);
}
void and32(BaseIndex src, RegisterID dest)
{
m_assembler.andl_mr(src.offset, src.base, src.index, src.scale, dest);
}
void and16(Address src, RegisterID dest)
{
m_assembler.andw_mr(src.offset, src.base, dest);
}
void and16(BaseIndex src, RegisterID dest)
{
m_assembler.andw_mr(src.offset, src.base, src.index, src.scale, dest);
}
void and32(TrustedImm32 imm, Address address)
{
m_assembler.andl_im(imm.m_value, address.offset, address.base);
}
void and32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.andl_im(imm.m_value, address.offset, address.base, address.index, address.scale);
}
void and16(TrustedImm32 imm, Address address)
{
m_assembler.andw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base);
}
void and16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.andw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base, address.index, address.scale);
}
void and8(TrustedImm32 imm, Address address)
{
m_assembler.andb_im(static_cast<int8_t>(imm.m_value), address.offset, address.base);
}
void and8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.andb_im(static_cast<int8_t>(imm.m_value), address.offset, address.base, address.index, address.scale);
}
void and32(RegisterID op1, RegisterID op2, RegisterID dest)
{
if (op1 == op2)
zeroExtend32ToWord(op1, dest);
else if (op1 == dest)
and32(op2, dest);
else {
move32IfNeeded(op2, dest);
and32(op1, dest);
}
}
void and32(Address op1, RegisterID op2, RegisterID dest)
{
if (op2 == dest)
and32(op1, dest);
else if (op1.base == dest) {
load32(op1, dest);
and32(op2, dest);
} else {
zeroExtend32ToWord(op2, dest);
and32(op1, dest);
}
}
void and32(RegisterID op1, Address op2, RegisterID dest)
{
and32(op2, op1, dest);
}
void and32(TrustedImm32 imm, RegisterID src, RegisterID dest)
{
move32IfNeeded(src, dest);
and32(imm, dest);
}
void countLeadingZeros32(RegisterID src, RegisterID dst)
{
if (supportsLZCNT()) {
m_assembler.lzcnt_rr(src, dst);
return;
}
m_assembler.bsr_rr(src, dst);
clz32AfterBsr(dst);
}
void countLeadingZeros32(Address src, RegisterID dst)
{
if (supportsLZCNT()) {
m_assembler.lzcnt_mr(src.offset, src.base, dst);
return;
}
m_assembler.bsr_mr(src.offset, src.base, dst);
clz32AfterBsr(dst);
}
void countTrailingZeros32(RegisterID src, RegisterID dst)
{
if (supportsBMI1()) {
m_assembler.tzcnt_rr(src, dst);
return;
}
m_assembler.bsf_rr(src, dst);
ctzAfterBsf<32>(dst);
}
void countPopulation32(Address src, RegisterID dst)
{
ASSERT(supportsCountPopulation());
m_assembler.popcnt_mr(src.offset, src.base, dst);
}
void countPopulation32(RegisterID src, RegisterID dst)
{
ASSERT(supportsCountPopulation());
m_assembler.popcnt_rr(src, dst);
}
void byteSwap32(RegisterID dst)
{
m_assembler.bswapl_r(dst);
}
void byteSwap16(RegisterID dst)
{
m_assembler.rolw_i8r(8, dst);
zeroExtend16To32(dst, dst);
}
#if CPU(X86_64)
void byteSwap64(RegisterID dst)
{
m_assembler.bswapq_r(dst);
}
#endif
// Only used for testing purposes.
void illegalInstruction()
{
m_assembler.illegalInstruction();
}
void lshift32(RegisterID shift_amount, RegisterID dest)
{
if (shift_amount == X86Registers::ecx)
m_assembler.shll_CLr(dest);
else {
ASSERT(shift_amount != dest);
// On x86 we can only shift by ecx; if asked to shift by another register we'll
// need rejig the shift amount into ecx first, and restore the registers afterwards.
// If we dest is ecx, then shift the swapped register!
swap(shift_amount, X86Registers::ecx);
m_assembler.shll_CLr(dest == X86Registers::ecx ? shift_amount : dest);
swap(shift_amount, X86Registers::ecx);
}
}
void lshift32(RegisterID src, RegisterID shift_amount, RegisterID dest)
{
ASSERT(shift_amount != dest);
move32IfNeeded(src, dest);
lshift32(shift_amount, dest);
}
void lshift32(TrustedImm32 imm, RegisterID dest)
{
m_assembler.shll_i8r(imm.m_value, dest);
}
void lshift32(RegisterID src, TrustedImm32 imm, RegisterID dest)
{
move32IfNeeded(src, dest);
lshift32(imm, dest);
}
void mul32(RegisterID src, RegisterID dest)
{
m_assembler.imull_rr(src, dest);
}
void mul32(RegisterID src1, RegisterID src2, RegisterID dest)
{
if (src2 == dest) {
m_assembler.imull_rr(src1, dest);
return;
}
move32IfNeeded(src1, dest);
m_assembler.imull_rr(src2, dest);
}
void mul32(Address src, RegisterID dest)
{
m_assembler.imull_mr(src.offset, src.base, dest);
}
void mul32(Address op1, RegisterID op2, RegisterID dest)
{
if (op2 == dest)
mul32(op1, dest);
else if (op1.base == dest) {
load32(op1, dest);
mul32(op2, dest);
} else {
zeroExtend32ToWord(op2, dest);
mul32(op1, dest);
}
}
void mul32(RegisterID src1, Address src2, RegisterID dest)
{
mul32(src2, src1, dest);
}
void mul32(TrustedImm32 imm, RegisterID src, RegisterID dest)
{
m_assembler.imull_i32r(src, imm.m_value, dest);
}
void x86ConvertToDoubleWord32()
{
m_assembler.cdq();
}
void x86ConvertToDoubleWord32(RegisterID eax, RegisterID edx)
{
ASSERT_UNUSED(eax, eax == X86Registers::eax);
ASSERT_UNUSED(edx, edx == X86Registers::edx);
x86ConvertToDoubleWord32();
}
void x86Div32(RegisterID denominator)
{
m_assembler.idivl_r(denominator);
}
void x86Div32(RegisterID eax, RegisterID edx, RegisterID denominator)
{
ASSERT_UNUSED(eax, eax == X86Registers::eax);
ASSERT_UNUSED(edx, edx == X86Registers::edx);
x86Div32(denominator);
}
void x86UDiv32(RegisterID denominator)
{
m_assembler.divl_r(denominator);
}
void x86UDiv32(RegisterID eax, RegisterID edx, RegisterID denominator)
{
ASSERT_UNUSED(eax, eax == X86Registers::eax);
ASSERT_UNUSED(edx, edx == X86Registers::edx);
x86UDiv32(denominator);
}
void neg32(RegisterID srcDest)
{
m_assembler.negl_r(srcDest);
}
void neg32(RegisterID src, RegisterID dest)
{
move32IfNeeded(src, dest);
m_assembler.negl_r(dest);
}
void neg32(Address srcDest)
{
m_assembler.negl_m(srcDest.offset, srcDest.base);
}
void neg32(BaseIndex srcDest)
{
m_assembler.negl_m(srcDest.offset, srcDest.base, srcDest.index, srcDest.scale);
}
void neg16(Address srcDest)
{
m_assembler.negw_m(srcDest.offset, srcDest.base);
}
void neg16(BaseIndex srcDest)
{
m_assembler.negw_m(srcDest.offset, srcDest.base, srcDest.index, srcDest.scale);
}
void neg8(Address srcDest)
{
m_assembler.negb_m(srcDest.offset, srcDest.base);
}
void neg8(BaseIndex srcDest)
{
m_assembler.negb_m(srcDest.offset, srcDest.base, srcDest.index, srcDest.scale);
}
void or32(RegisterID src, RegisterID dest)
{
m_assembler.orl_rr(src, dest);
}
void or32(TrustedImm32 imm, RegisterID dest)
{
m_assembler.orl_ir(imm.m_value, dest);
}
void or32(RegisterID src, Address dest)
{
m_assembler.orl_rm(src, dest.offset, dest.base);
}
void or32(RegisterID src, BaseIndex dest)
{
m_assembler.orl_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void or16(RegisterID src, Address dest)
{
m_assembler.orw_rm(src, dest.offset, dest.base);
}
void or16(RegisterID src, BaseIndex dest)
{
m_assembler.orw_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void or8(RegisterID src, Address dest)
{
m_assembler.orb_rm(src, dest.offset, dest.base);
}
void or8(RegisterID src, BaseIndex dest)
{
m_assembler.orb_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void or8(RegisterID src, AbsoluteAddress address)
{
move(TrustedImmPtr(address.m_ptr), scratchRegister());
or8(src, Address(scratchRegister()));
}
void or32(Address src, RegisterID dest)
{
m_assembler.orl_mr(src.offset, src.base, dest);
}
void or32(BaseIndex src, RegisterID dest)
{
m_assembler.orl_mr(src.offset, src.base, src.index, src.scale, dest);
}
void or32(TrustedImm32 imm, Address address)
{
m_assembler.orl_im(imm.m_value, address.offset, address.base);
}
void or32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.orl_im(imm.m_value, address.offset, address.base, address.index, address.scale);
}
void or16(TrustedImm32 imm, Address address)
{
m_assembler.orw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base);
}
void or16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.orw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base, address.index, address.scale);
}
void or8(TrustedImm32 imm, Address address)
{
m_assembler.orb_im(static_cast<int8_t>(imm.m_value), address.offset, address.base);
}
void or8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.orb_im(static_cast<int8_t>(imm.m_value), address.offset, address.base, address.index, address.scale);
}
void or8(TrustedImm32 imm, AbsoluteAddress address)
{
move(TrustedImmPtr(address.m_ptr), scratchRegister());
or8(imm, Address(scratchRegister()));
}
void or32(RegisterID op1, RegisterID op2, RegisterID dest)
{
if (op1 == op2)
zeroExtend32ToWord(op1, dest);
else if (op1 == dest)
or32(op2, dest);
else {
move32IfNeeded(op2, dest);
or32(op1, dest);
}
}
void or32(Address op1, RegisterID op2, RegisterID dest)
{
if (op2 == dest)
or32(op1, dest);
else if (op1.base == dest) {
load32(op1, dest);
or32(op2, dest);
} else {
zeroExtend32ToWord(op2, dest);
or32(op1, dest);
}
}
void or32(RegisterID op1, Address op2, RegisterID dest)
{
or32(op2, op1, dest);
}
void or32(TrustedImm32 imm, RegisterID src, RegisterID dest)
{
move32IfNeeded(src, dest);
or32(imm, dest);
}
void rshift32(RegisterID shift_amount, RegisterID dest)
{
if (shift_amount == X86Registers::ecx)
m_assembler.sarl_CLr(dest);
else {
ASSERT(shift_amount != dest);
// On x86 we can only shift by ecx; if asked to shift by another register we'll
// need rejig the shift amount into ecx first, and restore the registers afterwards.
// If we dest is ecx, then shift the swapped register!
swap(shift_amount, X86Registers::ecx);
m_assembler.sarl_CLr(dest == X86Registers::ecx ? shift_amount : dest);
swap(shift_amount, X86Registers::ecx);
}
}
void rshift32(RegisterID src, RegisterID shift_amount, RegisterID dest)
{
ASSERT(shift_amount != dest);
move32IfNeeded(src, dest);
rshift32(shift_amount, dest);
}
void rshift32(TrustedImm32 imm, RegisterID dest)
{
m_assembler.sarl_i8r(imm.m_value, dest);
}
void rshift32(RegisterID src, TrustedImm32 imm, RegisterID dest)
{
move32IfNeeded(src, dest);
rshift32(imm, dest);
}
void urshift32(RegisterID shift_amount, RegisterID dest)
{
if (shift_amount == X86Registers::ecx)
m_assembler.shrl_CLr(dest);
else {
ASSERT(shift_amount != dest);
// On x86 we can only shift by ecx; if asked to shift by another register we'll
// need rejig the shift amount into ecx first, and restore the registers afterwards.
// If we dest is ecx, then shift the swapped register!
swap(shift_amount, X86Registers::ecx);
m_assembler.shrl_CLr(dest == X86Registers::ecx ? shift_amount : dest);
swap(shift_amount, X86Registers::ecx);
}
}
void urshift32(RegisterID src, RegisterID shift_amount, RegisterID dest)
{
ASSERT(shift_amount != dest);
move32IfNeeded(src, dest);
urshift32(shift_amount, dest);
}
void urshift32(TrustedImm32 imm, RegisterID dest)
{
m_assembler.shrl_i8r(imm.m_value, dest);
}
void urshift32(RegisterID src, TrustedImm32 imm, RegisterID dest)
{
move32IfNeeded(src, dest);
urshift32(imm, dest);
}
void rotateRight32(TrustedImm32 imm, RegisterID dest)
{
m_assembler.rorl_i8r(imm.m_value, dest);
}
void rotateRight32(RegisterID src, RegisterID dest)
{
if (src == X86Registers::ecx)
m_assembler.rorl_CLr(dest);
else {
ASSERT(src != dest);
// Can only rotate by ecx, so we do some swapping if we see anything else.
swap(src, X86Registers::ecx);
m_assembler.rorl_CLr(dest == X86Registers::ecx ? src : dest);
swap(src, X86Registers::ecx);
}
}
void rotateLeft32(TrustedImm32 imm, RegisterID dest)
{
m_assembler.roll_i8r(imm.m_value, dest);
}
void rotateLeft32(RegisterID src, RegisterID dest)
{
if (src == X86Registers::ecx)
m_assembler.roll_CLr(dest);
else {
ASSERT(src != dest);
// Can only rotate by ecx, so we do some swapping if we see anything else.
swap(src, X86Registers::ecx);
m_assembler.roll_CLr(dest == X86Registers::ecx ? src : dest);
swap(src, X86Registers::ecx);
}
}
void sub32(RegisterID src, RegisterID dest)
{
m_assembler.subl_rr(src, dest);
}
void sub32(RegisterID left, RegisterID right, RegisterID dest)
{
if (dest == right) {
neg32(dest);
add32(left, dest);
return;
}
move(left, dest);
sub32(right, dest);
}
void sub32(RegisterID left, TrustedImm32 right, RegisterID dest)
{
move(left, dest);
sub32(right, dest);
}
void sub32(TrustedImm32 imm, RegisterID dest)
{
if (imm.m_value == 1)
m_assembler.dec_r(dest);
else
m_assembler.subl_ir(imm.m_value, dest);
}
void sub32(TrustedImm32 imm, Address address)
{
m_assembler.subl_im(imm.m_value, address.offset, address.base);
}
void sub16(TrustedImm32 imm, Address address)
{
m_assembler.subw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base);
}
void sub8(TrustedImm32 imm, Address address)
{
m_assembler.subb_im(static_cast<int8_t>(imm.m_value), address.offset, address.base);
}
void sub32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.subl_im(imm.m_value, address.offset, address.base, address.index, address.scale);
}
void sub16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.subw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base, address.index, address.scale);
}
void sub8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.subb_im(static_cast<int8_t>(imm.m_value), address.offset, address.base, address.index, address.scale);
}
void sub32(Address src, RegisterID dest)
{
m_assembler.subl_mr(src.offset, src.base, dest);
}
void sub32(BaseIndex src, RegisterID dest)
{
m_assembler.subl_mr(src.offset, src.base, src.index, src.scale, dest);
}
void sub32(RegisterID src, Address dest)
{
m_assembler.subl_rm(src, dest.offset, dest.base);
}
void sub16(RegisterID src, Address dest)
{
m_assembler.subw_rm(src, dest.offset, dest.base);
}
void sub8(RegisterID src, Address dest)
{
m_assembler.subb_rm(src, dest.offset, dest.base);
}
void sub32(RegisterID src, BaseIndex dest)
{
m_assembler.subl_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void sub16(RegisterID src, BaseIndex dest)
{
m_assembler.subw_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void sub8(RegisterID src, BaseIndex dest)
{
m_assembler.subb_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void xor32(RegisterID src, RegisterID dest)
{
m_assembler.xorl_rr(src, dest);
}
void xor32(TrustedImm32 imm, Address dest)
{
if (imm.m_value == -1)
m_assembler.notl_m(dest.offset, dest.base);
else
m_assembler.xorl_im(imm.m_value, dest.offset, dest.base);
}
void xor32(TrustedImm32 imm, BaseIndex dest)
{
if (imm.m_value == -1)
m_assembler.notl_m(dest.offset, dest.base, dest.index, dest.scale);
else
m_assembler.xorl_im(imm.m_value, dest.offset, dest.base, dest.index, dest.scale);
}
void xor16(TrustedImm32 imm, Address dest)
{
imm.m_value = static_cast<int16_t>(imm.m_value);
if (imm.m_value == -1)
m_assembler.notw_m(dest.offset, dest.base);
else
m_assembler.xorw_im(imm.m_value, dest.offset, dest.base);
}
void xor16(TrustedImm32 imm, BaseIndex dest)
{
imm.m_value = static_cast<int16_t>(imm.m_value);
if (imm.m_value == -1)
m_assembler.notw_m(dest.offset, dest.base, dest.index, dest.scale);
else
m_assembler.xorw_im(imm.m_value, dest.offset, dest.base, dest.index, dest.scale);
}
void xor8(TrustedImm32 imm, Address dest)
{
imm.m_value = static_cast<int8_t>(imm.m_value);
if (imm.m_value == -1)
m_assembler.notb_m(dest.offset, dest.base);
else
m_assembler.xorb_im(imm.m_value, dest.offset, dest.base);
}
void xor8(TrustedImm32 imm, BaseIndex dest)
{
imm.m_value = static_cast<int8_t>(imm.m_value);
if (imm.m_value == -1)
m_assembler.notb_m(dest.offset, dest.base, dest.index, dest.scale);
else
m_assembler.xorb_im(imm.m_value, dest.offset, dest.base, dest.index, dest.scale);
}
void xor32(TrustedImm32 imm, RegisterID dest)
{
if (imm.m_value == -1)
m_assembler.notl_r(dest);
else
m_assembler.xorl_ir(imm.m_value, dest);
}
void xor32(RegisterID src, Address dest)
{
m_assembler.xorl_rm(src, dest.offset, dest.base);
}
void xor32(RegisterID src, BaseIndex dest)
{
m_assembler.xorl_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void xor16(RegisterID src, Address dest)
{
m_assembler.xorw_rm(src, dest.offset, dest.base);
}
void xor16(RegisterID src, BaseIndex dest)
{
m_assembler.xorw_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void xor8(RegisterID src, Address dest)
{
m_assembler.xorb_rm(src, dest.offset, dest.base);
}
void xor8(RegisterID src, BaseIndex dest)
{
m_assembler.xorb_rm(src, dest.offset, dest.base, dest.index, dest.scale);
}
void xor32(Address src, RegisterID dest)
{
m_assembler.xorl_mr(src.offset, src.base, dest);
}
void xor32(BaseIndex src, RegisterID dest)
{
m_assembler.xorl_mr(src.offset, src.base, src.index, src.scale, dest);
}
void xor32(RegisterID op1, RegisterID op2, RegisterID dest)
{
if (op1 == op2)
move(TrustedImm32(0), dest);
else if (op1 == dest)
xor32(op2, dest);
else {
move32IfNeeded(op2, dest);
xor32(op1, dest);
}
}
void xor32(Address op1, RegisterID op2, RegisterID dest)
{
if (op2 == dest)
xor32(op1, dest);
else if (op1.base == dest) {
load32(op1, dest);
xor32(op2, dest);
} else {
zeroExtend32ToWord(op2, dest);
xor32(op1, dest);
}
}
void xor32(RegisterID op1, Address op2, RegisterID dest)
{
xor32(op2, op1, dest);
}
void xor32(TrustedImm32 imm, RegisterID src, RegisterID dest)
{
move32IfNeeded(src, dest);
xor32(imm, dest);
}
void not32(RegisterID srcDest)
{
m_assembler.notl_r(srcDest);
}
void not32(Address dest)
{
m_assembler.notl_m(dest.offset, dest.base);
}
void not32(BaseIndex dest)
{
m_assembler.notl_m(dest.offset, dest.base, dest.index, dest.scale);
}
void not16(Address dest)
{
m_assembler.notw_m(dest.offset, dest.base);
}
void not16(BaseIndex dest)
{
m_assembler.notw_m(dest.offset, dest.base, dest.index, dest.scale);
}
void not8(Address dest)
{
m_assembler.notb_m(dest.offset, dest.base);
}
void not8(BaseIndex dest)
{
m_assembler.notb_m(dest.offset, dest.base, dest.index, dest.scale);
}
void sqrtDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.sqrtsd_rr(src, dst);
}
void sqrtDouble(Address src, FPRegisterID dst)
{
m_assembler.sqrtsd_mr(src.offset, src.base, dst);
}
void sqrtFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.sqrtss_rr(src, dst);
}
void sqrtFloat(Address src, FPRegisterID dst)
{
m_assembler.sqrtss_mr(src.offset, src.base, dst);
}
void absDouble(FPRegisterID src, FPRegisterID dst)
{
ASSERT(src != dst);
static constexpr double negativeZeroConstant = -0.0;
loadDouble(TrustedImmPtr(&negativeZeroConstant), dst);
m_assembler.andnpd_rr(src, dst);
}
void negateDouble(FPRegisterID src, FPRegisterID dst)
{
ASSERT(src != dst);
static constexpr double negativeZeroConstant = -0.0;
loadDouble(TrustedImmPtr(&negativeZeroConstant), dst);
m_assembler.xorpd_rr(src, dst);
}
void ceilDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundsd_rr(src, dst, X86Assembler::RoundingType::TowardInfiniti);
}
void ceilDouble(Address src, FPRegisterID dst)
{
m_assembler.roundsd_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardInfiniti);
}
void ceilFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundss_rr(src, dst, X86Assembler::RoundingType::TowardInfiniti);
}
void ceilFloat(Address src, FPRegisterID dst)
{
m_assembler.roundss_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardInfiniti);
}
void floorDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundsd_rr(src, dst, X86Assembler::RoundingType::TowardNegativeInfiniti);
}
void floorDouble(Address src, FPRegisterID dst)
{
m_assembler.roundsd_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardNegativeInfiniti);
}
void floorFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundss_rr(src, dst, X86Assembler::RoundingType::TowardNegativeInfiniti);
}
void floorFloat(Address src, FPRegisterID dst)
{
m_assembler.roundss_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardNegativeInfiniti);
}
void roundTowardNearestIntDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundsd_rr(src, dst, X86Assembler::RoundingType::ToNearestWithTiesToEven);
}
void roundTowardNearestIntFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundss_rr(src, dst, X86Assembler::RoundingType::ToNearestWithTiesToEven);
}
void roundTowardZeroDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundsd_rr(src, dst, X86Assembler::RoundingType::TowardZero);
}
void roundTowardZeroDouble(Address src, FPRegisterID dst)
{
m_assembler.roundsd_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardZero);
}
void roundTowardZeroFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.roundss_rr(src, dst, X86Assembler::RoundingType::TowardZero);
}
void roundTowardZeroFloat(Address src, FPRegisterID dst)
{
m_assembler.roundss_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardZero);
}
// Memory access operations:
//
// Loads are of the form load(address, destination) and stores of the form
// store(source, address). The source for a store may be an TrustedImm32. Address
// operand objects to loads and store will be implicitly constructed if a
// register is passed.
void load32(Address address, RegisterID dest)
{
m_assembler.movl_mr(address.offset, address.base, dest);
}
void load32(BaseIndex address, RegisterID dest)
{
m_assembler.movl_mr(address.offset, address.base, address.index, address.scale, dest);
}
void load32WithUnalignedHalfWords(BaseIndex address, RegisterID dest)
{
load32(address, dest);
}
void load16Unaligned(Address address, RegisterID dest)
{
load16(address, dest);
}
void load16Unaligned(BaseIndex address, RegisterID dest)
{
load16(address, dest);
}
template<PtrTag tag>
static void repatchCompact(CodeLocationDataLabelCompact<tag> dataLabelCompact, int32_t value)
{
ASSERT(isCompactPtrAlignedAddressOffset(value));
AssemblerType_T::repatchCompact(dataLabelCompact.dataLocation(), value);
}
DataLabelCompact loadCompactWithAddressOffsetPatch(Address address, RegisterID dest)
{
padBeforePatch();
m_assembler.movl_mr_disp8(address.offset, address.base, dest);
return DataLabelCompact(this);
}
void load8(BaseIndex address, RegisterID dest)
{
m_assembler.movzbl_mr(address.offset, address.base, address.index, address.scale, dest);
}
void load8(Address address, RegisterID dest)
{
m_assembler.movzbl_mr(address.offset, address.base, dest);
}
void load8SignedExtendTo32(BaseIndex address, RegisterID dest)
{
m_assembler.movsbl_mr(address.offset, address.base, address.index, address.scale, dest);
}
void load8SignedExtendTo32(Address address, RegisterID dest)
{
m_assembler.movsbl_mr(address.offset, address.base, dest);
}
void zeroExtend8To32(RegisterID src, RegisterID dest)
{
m_assembler.movzbl_rr(src, dest);
}
void signExtend8To32(RegisterID src, RegisterID dest)
{
m_assembler.movsbl_rr(src, dest);
}
void load16(Address address, RegisterID dest)
{
m_assembler.movzwl_mr(address.offset, address.base, dest);
}
void load16(BaseIndex address, RegisterID dest)
{
m_assembler.movzwl_mr(address.offset, address.base, address.index, address.scale, dest);
}
void load16SignedExtendTo32(BaseIndex address, RegisterID dest)
{
m_assembler.movswl_mr(address.offset, address.base, address.index, address.scale, dest);
}
void load16SignedExtendTo32(Address address, RegisterID dest)
{
m_assembler.movswl_mr(address.offset, address.base, dest);
}
void loadPair32(RegisterID src, RegisterID dest1, RegisterID dest2)
{
loadPair32(src, TrustedImm32(0), dest1, dest2);
}
void loadPair32(RegisterID src, TrustedImm32 offset, RegisterID dest1, RegisterID dest2)
{
ASSERT(dest1 != dest2); // If it is the same, ldp becomes illegal instruction.
if (src == dest1) {
load32(Address(src, offset.m_value + 4), dest2);
load32(Address(src, offset.m_value), dest1);
} else {
load32(Address(src, offset.m_value), dest1);
load32(Address(src, offset.m_value + 4), dest2);
}
}
void zeroExtend16To32(RegisterID src, RegisterID dest)
{
m_assembler.movzwl_rr(src, dest);
}
void signExtend16To32(RegisterID src, RegisterID dest)
{
m_assembler.movswl_rr(src, dest);
}
void store32(RegisterID src, Address address)
{
m_assembler.movl_rm(src, address.offset, address.base);
}
void store32(RegisterID src, BaseIndex address)
{
m_assembler.movl_rm(src, address.offset, address.base, address.index, address.scale);
}
void store32(TrustedImm32 imm, Address address)
{
m_assembler.movl_i32m(imm.m_value, address.offset, address.base);
}
void store32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.movl_i32m(imm.m_value, address.offset, address.base, address.index, address.scale);
}
void store8(TrustedImm32 imm, Address address)
{
TrustedImm32 imm8(static_cast<int8_t>(imm.m_value));
m_assembler.movb_i8m(imm8.m_value, address.offset, address.base);
}
void store8(TrustedImm32 imm, BaseIndex address)
{
TrustedImm32 imm8(static_cast<int8_t>(imm.m_value));
m_assembler.movb_i8m(imm8.m_value, address.offset, address.base, address.index, address.scale);
}
void storePair32(RegisterID src1, RegisterID src2, RegisterID dest)
{
storePair32(src1, src2, dest, TrustedImm32(0));
}
void storePair32(RegisterID src1, RegisterID src2, RegisterID dest, TrustedImm32 offset)
{
store32(src1, Address(dest, offset.m_value));
store32(src2, Address(dest, offset.m_value + 4));
}
static ALWAYS_INLINE RegisterID getUnusedRegister(BaseIndex address)
{
if (address.base != X86Registers::eax && address.index != X86Registers::eax)
return X86Registers::eax;
if (address.base != X86Registers::ebx && address.index != X86Registers::ebx)
return X86Registers::ebx;
ASSERT(address.base != X86Registers::ecx && address.index != X86Registers::ecx);
return X86Registers::ecx;
}
static ALWAYS_INLINE RegisterID getUnusedRegister(Address address)
{
if (address.base != X86Registers::eax)
return X86Registers::eax;
ASSERT(address.base != X86Registers::edx);
return X86Registers::edx;
}
void store8(RegisterID src, BaseIndex address)
{
#if CPU(X86)
// On 32-bit x86 we can only store from the first 4 registers;
// esp..edi are mapped to the 'h' registers!
if (src >= 4) {
// Pick a temporary register.
RegisterID temp = getUnusedRegister(address);
// Swap to the temporary register to perform the store.
swap(src, temp);
m_assembler.movb_rm(temp, address.offset, address.base, address.index, address.scale);
swap(src, temp);
return;
}
#endif
m_assembler.movb_rm(src, address.offset, address.base, address.index, address.scale);
}
void store8(RegisterID src, Address address)
{
#if CPU(X86)
// On 32-bit x86 we can only store from the first 4 registers;
// esp..edi are mapped to the 'h' registers!
if (src >= 4) {
// Pick a temporary register.
RegisterID temp = getUnusedRegister(address);
// Swap to the temporary register to perform the store.
swap(src, temp);
m_assembler.movb_rm(temp, address.offset, address.base);
swap(src, temp);
return;
}
#endif
m_assembler.movb_rm(src, address.offset, address.base);
}
void store16(RegisterID src, BaseIndex address)
{
m_assembler.movw_rm(src, address.offset, address.base, address.index, address.scale);
}
void store16(RegisterID src, Address address)
{
m_assembler.movw_rm(src, address.offset, address.base);
}
void store16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.movw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base, address.index, address.scale);
}
void store16(TrustedImm32 imm, Address address)
{
m_assembler.movw_im(static_cast<int16_t>(imm.m_value), address.offset, address.base);
}
// Floating-point operation:
//
void moveDouble(FPRegisterID src, FPRegisterID dest)
{
if (src != dest)
m_assembler.movaps_rr(src, dest);
}
void loadDouble(TrustedImmPtr address, FPRegisterID dest)
{
#if CPU(X86)
m_assembler.movsd_mr(address.asPtr(), dest);
#else
move(address, scratchRegister());
loadDouble(Address(scratchRegister()), dest);
#endif
}
void loadDouble(Address address, FPRegisterID dest)
{
m_assembler.movsd_mr(address.offset, address.base, dest);
}
void loadDouble(BaseIndex address, FPRegisterID dest)
{
m_assembler.movsd_mr(address.offset, address.base, address.index, address.scale, dest);
}
void loadFloat(TrustedImmPtr address, FPRegisterID dest)
{
#if CPU(X86)
m_assembler.movss_mr(address.asPtr(), dest);
#else
move(address, scratchRegister());
loadFloat(Address(scratchRegister()), dest);
#endif
}
void loadFloat(Address address, FPRegisterID dest)
{
m_assembler.movss_mr(address.offset, address.base, dest);
}
void loadFloat(BaseIndex address, FPRegisterID dest)
{
m_assembler.movss_mr(address.offset, address.base, address.index, address.scale, dest);
}
void storeDouble(FPRegisterID src, Address address)
{
m_assembler.movsd_rm(src, address.offset, address.base);
}
void storeDouble(FPRegisterID src, BaseIndex address)
{
m_assembler.movsd_rm(src, address.offset, address.base, address.index, address.scale);
}
void storeFloat(FPRegisterID src, Address address)
{
m_assembler.movss_rm(src, address.offset, address.base);
}
void storeFloat(FPRegisterID src, BaseIndex address)
{
m_assembler.movss_rm(src, address.offset, address.base, address.index, address.scale);
}
void convertDoubleToFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.cvtsd2ss_rr(src, dst);
}
void convertDoubleToFloat(Address address, FPRegisterID dst)
{
m_assembler.cvtsd2ss_mr(address.offset, address.base, dst);
}
void convertFloatToDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.cvtss2sd_rr(src, dst);
}
void convertFloatToDouble(Address address, FPRegisterID dst)
{
m_assembler.cvtss2sd_mr(address.offset, address.base, dst);
}
void addDouble(FPRegisterID src, FPRegisterID dest)
{
addDouble(src, dest, dest);
}
void addDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vaddsd_rr(op1, op2, dest);
else {
if (op1 == dest)
m_assembler.addsd_rr(op2, dest);
else {
moveDouble(op2, dest);
m_assembler.addsd_rr(op1, dest);
}
}
}
void addDouble(Address src, FPRegisterID dest)
{
addDouble(src, dest, dest);
}
void addDouble(Address op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vaddsd_mr(op1.offset, op1.base, op2, dest);
else {
if (op2 == dest) {
m_assembler.addsd_mr(op1.offset, op1.base, dest);
return;
}
loadDouble(op1, dest);
addDouble(op2, dest);
}
}
void addDouble(FPRegisterID op1, Address op2, FPRegisterID dest)
{
addDouble(op2, op1, dest);
}
void addDouble(BaseIndex op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vaddsd_mr(op1.offset, op1.base, op1.index, op1.scale, op2, dest);
else {
if (op2 == dest) {
m_assembler.addsd_mr(op1.offset, op1.base, op1.index, op1.scale, dest);
return;
}
loadDouble(op1, dest);
addDouble(op2, dest);
}
}
void addFloat(FPRegisterID src, FPRegisterID dest)
{
addFloat(src, dest, dest);
}
void addFloat(Address src, FPRegisterID dest)
{
addFloat(src, dest, dest);
}
void addFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vaddss_rr(op1, op2, dest);
else {
if (op1 == dest)
m_assembler.addss_rr(op2, dest);
else {
moveDouble(op2, dest);
m_assembler.addss_rr(op1, dest);
}
}
}
void addFloat(Address op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vaddss_mr(op1.offset, op1.base, op2, dest);
else {
if (op2 == dest) {
m_assembler.addss_mr(op1.offset, op1.base, dest);
return;
}
loadFloat(op1, dest);
addFloat(op2, dest);
}
}
void addFloat(FPRegisterID op1, Address op2, FPRegisterID dest)
{
addFloat(op2, op1, dest);
}
void addFloat(BaseIndex op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vaddss_mr(op1.offset, op1.base, op1.index, op1.scale, op2, dest);
else {
if (op2 == dest) {
m_assembler.addss_mr(op1.offset, op1.base, op1.index, op1.scale, dest);
return;
}
loadFloat(op1, dest);
addFloat(op2, dest);
}
}
void divDouble(FPRegisterID src, FPRegisterID dest)
{
m_assembler.divsd_rr(src, dest);
}
void divDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest)
{
// B := A / B is invalid.
ASSERT(op1 == dest || op2 != dest);
moveDouble(op1, dest);
divDouble(op2, dest);
}
void divDouble(Address src, FPRegisterID dest)
{
m_assembler.divsd_mr(src.offset, src.base, dest);
}
void divFloat(FPRegisterID src, FPRegisterID dest)
{
m_assembler.divss_rr(src, dest);
}
void divFloat(Address src, FPRegisterID dest)
{
m_assembler.divss_mr(src.offset, src.base, dest);
}
void subDouble(FPRegisterID src, FPRegisterID dest)
{
subDouble(dest, src, dest);
}
void subDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vsubsd_rr(op1, op2, dest);
else {
// B := A - B is invalid.
ASSERT(op1 == dest || op2 != dest);
moveDouble(op1, dest);
m_assembler.subsd_rr(op2, dest);
}
}
void subDouble(FPRegisterID op1, Address op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vsubsd_mr(op1, op2.offset, op2.base, dest);
else {
moveDouble(op1, dest);
m_assembler.subsd_mr(op2.offset, op2.base, dest);
}
}
void subDouble(FPRegisterID op1, BaseIndex op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vsubsd_mr(op1, op2.offset, op2.base, op2.index, op2.scale, dest);
else {
moveDouble(op1, dest);
m_assembler.subsd_mr(op2.offset, op2.base, op2.index, op2.scale, dest);
}
}
void subDouble(Address src, FPRegisterID dest)
{
subDouble(dest, src, dest);
}
void subFloat(FPRegisterID src, FPRegisterID dest)
{
subFloat(dest, src, dest);
}
void subFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vsubss_rr(op1, op2, dest);
else {
// B := A - B is invalid.
ASSERT(op1 == dest || op2 != dest);
moveDouble(op1, dest);
m_assembler.subss_rr(op2, dest);
}
}
void subFloat(FPRegisterID op1, Address op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vsubss_mr(op1, op2.offset, op2.base, dest);
else {
moveDouble(op1, dest);
m_assembler.subss_mr(op2.offset, op2.base, dest);
}
}
void subFloat(FPRegisterID op1, BaseIndex op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vsubss_mr(op1, op2.offset, op2.base, op2.index, op2.scale, dest);
else {
moveDouble(op1, dest);
m_assembler.subss_mr(op2.offset, op2.base, op2.index, op2.scale, dest);
}
}
void subFloat(Address src, FPRegisterID dest)
{
subFloat(dest, src, dest);
}
void mulDouble(FPRegisterID src, FPRegisterID dest)
{
mulDouble(src, dest, dest);
}
void mulDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vmulsd_rr(op1, op2, dest);
else {
if (op1 == dest)
m_assembler.mulsd_rr(op2, dest);
else {
moveDouble(op2, dest);
m_assembler.mulsd_rr(op1, dest);
}
}
}
void mulDouble(Address src, FPRegisterID dest)
{
mulDouble(src, dest, dest);
}
void mulDouble(Address op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vmulsd_mr(op1.offset, op1.base, op2, dest);
else {
if (op2 == dest) {
m_assembler.mulsd_mr(op1.offset, op1.base, dest);
return;
}
loadDouble(op1, dest);
mulDouble(op2, dest);
}
}
void mulDouble(FPRegisterID op1, Address op2, FPRegisterID dest)
{
return mulDouble(op2, op1, dest);
}
void mulDouble(BaseIndex op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vmulsd_mr(op1.offset, op1.base, op1.index, op1.scale, op2, dest);
else {
if (op2 == dest) {
m_assembler.mulsd_mr(op1.offset, op1.base, op1.index, op1.scale, dest);
return;
}
loadDouble(op1, dest);
mulDouble(op2, dest);
}
}
void mulFloat(FPRegisterID src, FPRegisterID dest)
{
mulFloat(src, dest, dest);
}
void mulFloat(Address src, FPRegisterID dest)
{
mulFloat(src, dest, dest);
}
void mulFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vmulss_rr(op1, op2, dest);
else {
if (op1 == dest)
m_assembler.mulss_rr(op2, dest);
else {
moveDouble(op2, dest);
m_assembler.mulss_rr(op1, dest);
}
}
}
void mulFloat(Address op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vmulss_mr(op1.offset, op1.base, op2, dest);
else {
if (op2 == dest) {
m_assembler.mulss_mr(op1.offset, op1.base, dest);
return;
}
loadFloat(op1, dest);
mulFloat(op2, dest);
}
}
void mulFloat(FPRegisterID op1, Address op2, FPRegisterID dest)
{
mulFloat(op2, op1, dest);
}
void mulFloat(BaseIndex op1, FPRegisterID op2, FPRegisterID dest)
{
if (supportsAVX())
m_assembler.vmulss_mr(op1.offset, op1.base, op1.index, op1.scale, op2, dest);
else {
if (op2 == dest) {
m_assembler.mulss_mr(op1.offset, op1.base, op1.index, op1.scale, dest);
return;
}
loadFloat(op1, dest);
mulFloat(op2, dest);
}
}
void andDouble(FPRegisterID src, FPRegisterID dst)
{
// ANDPS is defined on 128bits and is shorter than ANDPD.
m_assembler.andps_rr(src, dst);
}
void andDouble(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst)
{
if (src1 == dst)
andDouble(src2, dst);
else {
moveDouble(src2, dst);
andDouble(src1, dst);
}
}
void andFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.andps_rr(src, dst);
}
void andFloat(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst)
{
if (src1 == dst)
andFloat(src2, dst);
else {
moveDouble(src2, dst);
andFloat(src1, dst);
}
}
void orDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.orps_rr(src, dst);
}
void orDouble(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst)
{
if (src1 == dst)
orDouble(src2, dst);
else {
moveDouble(src2, dst);
orDouble(src1, dst);
}
}
void orFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.orps_rr(src, dst);
}
void orFloat(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst)
{
if (src1 == dst)
orFloat(src2, dst);
else {
moveDouble(src2, dst);
orFloat(src1, dst);
}
}
void xorDouble(FPRegisterID src, FPRegisterID dst)
{
m_assembler.xorps_rr(src, dst);
}
void xorDouble(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst)
{
if (src1 == dst)
xorDouble(src2, dst);
else {
moveDouble(src2, dst);
xorDouble(src1, dst);
}
}
void xorFloat(FPRegisterID src, FPRegisterID dst)
{
m_assembler.xorps_rr(src, dst);
}
void xorFloat(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst)
{
if (src1 == dst)
xorFloat(src2, dst);
else {
moveDouble(src2, dst);
xorFloat(src1, dst);
}
}
void convertInt32ToDouble(RegisterID src, FPRegisterID dest)
{
m_assembler.cvtsi2sd_rr(src, dest);
}
void convertInt32ToDouble(Address src, FPRegisterID dest)
{
m_assembler.cvtsi2sd_mr(src.offset, src.base, dest);
}
void convertInt32ToFloat(RegisterID src, FPRegisterID dest)
{
m_assembler.cvtsi2ss_rr(src, dest);
}
void convertInt32ToFloat(Address src, FPRegisterID dest)
{
m_assembler.cvtsi2ss_mr(src.offset, src.base, dest);
}
Jump branchDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right)
{
if (cond & DoubleConditionBitInvert)
m_assembler.ucomisd_rr(left, right);
else
m_assembler.ucomisd_rr(right, left);
return jumpAfterFloatingPointCompare(cond, left, right);
}
Jump branchFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right)
{
if (cond & DoubleConditionBitInvert)
m_assembler.ucomiss_rr(left, right);
else
m_assembler.ucomiss_rr(right, left);
return jumpAfterFloatingPointCompare(cond, left, right);
}
void compareDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID dest)
{
floatingPointCompare(cond, left, right, dest, [this] (FPRegisterID arg1, FPRegisterID arg2) {
m_assembler.ucomisd_rr(arg1, arg2);
});
}
void compareFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID dest)
{
floatingPointCompare(cond, left, right, dest, [this] (FPRegisterID arg1, FPRegisterID arg2) {
m_assembler.ucomiss_rr(arg1, arg2);
});
}
// Truncates 'src' to an integer, and places the resulting 'dest'.
// If the result is not representable as a 32 bit value, branch.
// May also branch for some values that are representable in 32 bits
// (specifically, in this case, INT_MIN).
enum BranchTruncateType { BranchIfTruncateFailed, BranchIfTruncateSuccessful };
Jump branchTruncateDoubleToInt32(FPRegisterID src, RegisterID dest, BranchTruncateType branchType = BranchIfTruncateFailed)
{
m_assembler.cvttsd2si_rr(src, dest);
return branch32(branchType ? NotEqual : Equal, dest, TrustedImm32(0x80000000));
}
void truncateDoubleToInt32(FPRegisterID src, RegisterID dest)
{
m_assembler.cvttsd2si_rr(src, dest);
}
void truncateFloatToInt32(FPRegisterID src, RegisterID dest)
{
m_assembler.cvttss2si_rr(src, dest);
}
// Convert 'src' to an integer, and places the resulting 'dest'.
// If the result is not representable as a 32 bit value, branch.
// May also branch for some values that are representable in 32 bits
// (specifically, in this case, 0).
void branchConvertDoubleToInt32(FPRegisterID src, RegisterID dest, JumpList& failureCases, FPRegisterID fpTemp, bool negZeroCheck = true)
{
m_assembler.cvttsd2si_rr(src, dest);
// If the result is zero, it might have been -0.0, and the double comparison won't catch this!
#if CPU(X86_64)
if (negZeroCheck) {
Jump valueIsNonZero = branchTest32(NonZero, dest);
m_assembler.movmskpd_rr(src, scratchRegister());
failureCases.append(branchTest32(NonZero, scratchRegister(), TrustedImm32(1)));
valueIsNonZero.link(this);
}
#else
if (negZeroCheck)
failureCases.append(branchTest32(Zero, dest));
#endif
// Convert the integer result back to float & compare to the original value - if not equal or unordered (NaN) then jump.
convertInt32ToDouble(dest, fpTemp);
m_assembler.ucomisd_rr(fpTemp, src);
failureCases.append(m_assembler.jp());
failureCases.append(m_assembler.jne());
}
void moveZeroToDouble(FPRegisterID reg)
{
m_assembler.xorps_rr(reg, reg);
}
Jump branchDoubleNonZero(FPRegisterID reg, FPRegisterID scratch)
{
m_assembler.xorpd_rr(scratch, scratch);
return branchDouble(DoubleNotEqualAndOrdered, reg, scratch);
}
Jump branchDoubleZeroOrNaN(FPRegisterID reg, FPRegisterID scratch)
{
m_assembler.xorpd_rr(scratch, scratch);
return branchDouble(DoubleEqualOrUnordered, reg, scratch);
}
void lshiftPacked(TrustedImm32 imm, XMMRegisterID reg)
{
m_assembler.psllq_i8r(imm.m_value, reg);
}
void rshiftPacked(TrustedImm32 imm, XMMRegisterID reg)
{
m_assembler.psrlq_i8r(imm.m_value, reg);
}
void orPacked(XMMRegisterID src, XMMRegisterID dst)
{
m_assembler.por_rr(src, dst);
}
void move32ToFloat(RegisterID src, XMMRegisterID dst)
{
m_assembler.movd_rr(src, dst);
}
void moveFloatTo32(XMMRegisterID src, RegisterID dst)
{
m_assembler.movd_rr(src, dst);
}
// Stack manipulation operations:
//
// The ABI is assumed to provide a stack abstraction to memory,
// containing machine word sized units of data. Push and pop
// operations add and remove a single register sized unit of data
// to or from the stack. Peek and poke operations read or write
// values on the stack, without moving the current stack position.
void pop(RegisterID dest)
{
m_assembler.pop_r(dest);
}
void push(RegisterID src)
{
m_assembler.push_r(src);
}
void push(Address address)
{
m_assembler.push_m(address.offset, address.base);
}
void push(TrustedImm32 imm)
{
m_assembler.push_i32(imm.m_value);
}
void popPair(RegisterID dest1, RegisterID dest2)
{
pop(dest2);
pop(dest1);
}
void pushPair(RegisterID src1, RegisterID src2)
{
push(src1);
push(src2);
}
// Register move operations:
//
// Move values in registers.
void move(TrustedImm32 imm, RegisterID dest)
{
// Note: on 64-bit the TrustedImm32 value is zero extended into the register, it
// may be useful to have a separate version that sign extends the value?
if (!imm.m_value)
m_assembler.xorl_rr(dest, dest);
else
m_assembler.movl_i32r(imm.m_value, dest);
}
#if CPU(X86_64)
void move(RegisterID src, RegisterID dest)
{
// Note: on 64-bit this is is a full register move; perhaps it would be
// useful to have separate move32 & movePtr, with move32 zero extending?
if (src != dest)
m_assembler.movq_rr(src, dest);
}
void move(TrustedImmPtr imm, RegisterID dest)
{
if (!imm.m_value)
m_assembler.xorq_rr(dest, dest);
else
m_assembler.movq_i64r(imm.asIntptr(), dest);
}
void move(TrustedImm64 imm, RegisterID dest)
{
if (!imm.m_value)
m_assembler.xorq_rr(dest, dest);
else
m_assembler.movq_i64r(imm.m_value, dest);
}
void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest)
{
if (cond & DoubleConditionBitInvert)
m_assembler.ucomisd_rr(left, right);
else
m_assembler.ucomisd_rr(right, left);
moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest);
}
void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest)
{
if (thenCase != dest && elseCase != dest) {
move(elseCase, dest);
elseCase = dest;
}
RegisterID src;
if (elseCase == dest)
src = thenCase;
else {
cond = invert(cond);
src = elseCase;
}
if (cond & DoubleConditionBitInvert)
m_assembler.ucomisd_rr(left, right);
else
m_assembler.ucomisd_rr(right, left);
moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest);
}
void moveConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest)
{
if (cond & DoubleConditionBitInvert)
m_assembler.ucomiss_rr(left, right);
else
m_assembler.ucomiss_rr(right, left);
moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest);
}
void moveConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest)
{
if (thenCase != dest && elseCase != dest) {
move(elseCase, dest);
elseCase = dest;
}
RegisterID src;
if (elseCase == dest)
src = thenCase;
else {
cond = invert(cond);
src = elseCase;
}
if (cond & DoubleConditionBitInvert)
m_assembler.ucomiss_rr(left, right);
else
m_assembler.ucomiss_rr(right, left);
moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest);
}
void swap(RegisterID reg1, RegisterID reg2)
{
if (reg1 != reg2)
m_assembler.xchgq_rr(reg1, reg2);
}
void swap(FPRegisterID reg1, FPRegisterID reg2)
{
if (reg1 == reg2)
return;
// FIXME: This is kinda a hack since we don't use xmm7 as a temp.
ASSERT(reg1 != FPRegisterID::xmm7);
ASSERT(reg2 != FPRegisterID::xmm7);
moveDouble(reg1, FPRegisterID::xmm7);
moveDouble(reg2, reg1);
moveDouble(FPRegisterID::xmm7, reg2);
}
void signExtend32ToPtr(TrustedImm32 imm, RegisterID dest)
{
if (!imm.m_value)
m_assembler.xorq_rr(dest, dest);
else
m_assembler.mov_i32r(imm.m_value, dest);
}
void signExtend32ToPtr(RegisterID src, RegisterID dest)
{
m_assembler.movsxd_rr(src, dest);
}
void zeroExtend32ToWord(RegisterID src, RegisterID dest)
{
m_assembler.movl_rr(src, dest);
}
void zeroExtend32ToWord(TrustedImm32 src, RegisterID dest)
{
m_assembler.movl_i32r(src.m_value, dest);
}
#else
void move(RegisterID src, RegisterID dest)
{
if (src != dest)
m_assembler.movl_rr(src, dest);
}
void move(TrustedImmPtr imm, RegisterID dest)
{
if (!imm.m_value)
m_assembler.xorl_rr(dest, dest);
else
m_assembler.movl_i32r(imm.asIntptr(), dest);
}
// Only here for templates!
void move(TrustedImm64, RegisterID)
{
UNREACHABLE_FOR_PLATFORM();
}
void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest)
{
if (cond & DoubleConditionBitInvert)
m_assembler.ucomisd_rr(left, right);
else
m_assembler.ucomisd_rr(right, left);
if (cond == DoubleEqualAndOrdered) {
if (left == right) {
m_assembler.cmovnpl_rr(src, dest);
return;
}
Jump isUnordered(m_assembler.jp());
m_assembler.cmovel_rr(src, dest);
isUnordered.link(this);
return;
}
if (cond == DoubleNotEqualOrUnordered) {
if (left == right) {
m_assembler.cmovpl_rr(src, dest);
return;
}
m_assembler.cmovpl_rr(src, dest);
m_assembler.cmovnel_rr(src, dest);
return;
}
ASSERT(!(cond & DoubleConditionBitSpecial));
m_assembler.cmovl_rr(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits), src, dest);
}
void swap(RegisterID reg1, RegisterID reg2)
{
if (reg1 != reg2)
m_assembler.xchgl_rr(reg1, reg2);
}
void swap(FPRegisterID reg1, FPRegisterID reg2)
{
if (reg1 == reg2)
return;
// FIXME: This is kinda a hack since we don't use xmm7 as a temp.
ASSERT(reg1 != FPRegisterID::xmm7);
ASSERT(reg2 != FPRegisterID::xmm7);
moveDouble(reg1, FPRegisterID::xmm7);
moveDouble(reg2, reg1);
moveDouble(FPRegisterID::xmm7, reg2);
}
void signExtend32ToPtr(RegisterID src, RegisterID dest)
{
move(src, dest);
}
void zeroExtend32ToWord(RegisterID src, RegisterID dest)
{
move(src, dest);
}
#endif
void swap32(RegisterID src, RegisterID dest)
{
m_assembler.xchgl_rr(src, dest);
}
void swap32(RegisterID src, Address dest)
{
m_assembler.xchgl_rm(src, dest.offset, dest.base);
}
void moveConditionally32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID src, RegisterID dest)
{
m_assembler.cmpl_rr(right, left);
cmov(x86Condition(cond), src, dest);
}
void moveConditionally32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest)
{
m_assembler.cmpl_rr(right, left);
if (thenCase != dest && elseCase != dest) {
move(elseCase, dest);
elseCase = dest;
}
if (elseCase == dest)
cmov(x86Condition(cond), thenCase, dest);
else
cmov(x86Condition(invert(cond)), elseCase, dest);
}
void moveConditionally32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID thenCase, RegisterID elseCase, RegisterID dest)
{
if (!right.m_value) {
if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) {
moveConditionallyTest32(*resultCondition, left, left, thenCase, elseCase, dest);
return;
}
}
m_assembler.cmpl_ir(right.m_value, left);
if (thenCase != dest && elseCase != dest) {
move(elseCase, dest);
elseCase = dest;
}
if (elseCase == dest)
cmov(x86Condition(cond), thenCase, dest);
else
cmov(x86Condition(invert(cond)), elseCase, dest);
}
void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, RegisterID mask, RegisterID src, RegisterID dest)
{
m_assembler.testl_rr(testReg, mask);
cmov(x86Condition(cond), src, dest);
}
void moveConditionallyTest32(ResultCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest)
{
ASSERT(isInvertible(cond));
ASSERT_WITH_MESSAGE(cond != Overflow, "TEST does not set the Overflow Flag.");
m_assembler.testl_rr(right, left);
if (thenCase != dest && elseCase != dest) {
move(elseCase, dest);
elseCase = dest;
}
if (elseCase == dest)
cmov(x86Condition(cond), thenCase, dest);
else
cmov(x86Condition(invert(cond)), elseCase, dest);
}
void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, TrustedImm32 mask, RegisterID src, RegisterID dest)
{
test32(testReg, mask);
cmov(x86Condition(cond), src, dest);
}
void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, TrustedImm32 mask, RegisterID thenCase, RegisterID elseCase, RegisterID dest)
{
ASSERT(isInvertible(cond));
ASSERT_WITH_MESSAGE(cond != Overflow, "TEST does not set the Overflow Flag.");
test32(testReg, mask);
if (thenCase != dest && elseCase != dest) {
move(elseCase, dest);
elseCase = dest;
}
if (elseCase == dest)
cmov(x86Condition(cond), thenCase, dest);
else
cmov(x86Condition(invert(cond)), elseCase, dest);
}
template<typename LeftType, typename RightType>
void moveDoubleConditionally32(RelationalCondition cond, LeftType left, RightType right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest)
{
static_assert(!std::is_same<LeftType, FPRegisterID>::value && !std::is_same<RightType, FPRegisterID>::value, "One of the tested argument could be aliased on dest. Use moveDoubleConditionallyDouble().");
if (thenCase != dest && elseCase != dest) {
moveDouble(elseCase, dest);
elseCase = dest;
}
if (elseCase == dest) {
Jump falseCase = branch32(invert(cond), left, right);
moveDouble(thenCase, dest);
falseCase.link(this);
} else {
Jump trueCase = branch32(cond, left, right);
moveDouble(elseCase, dest);
trueCase.link(this);
}
}
template<typename TestType, typename MaskType>
void moveDoubleConditionallyTest32(ResultCondition cond, TestType test, MaskType mask, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest)
{
static_assert(!std::is_same<TestType, FPRegisterID>::value && !std::is_same<MaskType, FPRegisterID>::value, "One of the tested argument could be aliased on dest. Use moveDoubleConditionallyDouble().");
if (elseCase == dest && isInvertible(cond)) {
Jump falseCase = branchTest32(invert(cond), test, mask);
moveDouble(thenCase, dest);
falseCase.link(this);
} else if (thenCase == dest) {
Jump trueCase = branchTest32(cond, test, mask);
moveDouble(elseCase, dest);
trueCase.link(this);
}
Jump trueCase = branchTest32(cond, test, mask);
moveDouble(elseCase, dest);
Jump falseCase = jump();
trueCase.link(this);
moveDouble(thenCase, dest);
falseCase.link(this);
}
void moveDoubleConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest)
{
if (elseCase == dest) {
Jump falseCase = branchDouble(invert(cond), left, right);
moveDouble(thenCase, dest);
falseCase.link(this);
} else if (thenCase == dest) {
Jump trueCase = branchDouble(cond, left, right);
moveDouble(elseCase, dest);
trueCase.link(this);
} else {
Jump trueCase = branchDouble(cond, left, right);
moveDouble(elseCase, dest);
Jump falseCase = jump();
trueCase.link(this);
moveDouble(thenCase, dest);
falseCase.link(this);
}
}
void moveDoubleConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest)
{
if (elseCase == dest) {
Jump falseCase = branchFloat(invert(cond), left, right);
moveDouble(thenCase, dest);
falseCase.link(this);
} else if (thenCase == dest) {
Jump trueCase = branchFloat(cond, left, right);
moveDouble(elseCase, dest);
trueCase.link(this);
} else {
Jump trueCase = branchFloat(cond, left, right);
moveDouble(elseCase, dest);
Jump falseCase = jump();
trueCase.link(this);
moveDouble(thenCase, dest);
falseCase.link(this);
}
}
// Forwards / external control flow operations:
//
// This set of jump and conditional branch operations return a Jump
// object which may linked at a later point, allow forwards jump,
// or jumps that will require external linkage (after the code has been
// relocated).
//
// For branches, signed <, >, <= and >= are denoted as l, g, le, and ge
// respecitvely, for unsigned comparisons the names b, a, be, and ae are
// used (representing the names 'below' and 'above').
//
// Operands to the comparision are provided in the expected order, e.g.
// jle32(reg1, TrustedImm32(5)) will branch if the value held in reg1, when
// treated as a signed 32bit value, is less than or equal to 5.
//
// jz and jnz test whether the first operand is equal to zero, and take
// an optional second operand of a mask under which to perform the test.
public:
Jump branch8(RelationalCondition cond, Address left, TrustedImm32 right)
{
TrustedImm32 right8(static_cast<int8_t>(right.m_value));
m_assembler.cmpb_im(right8.m_value, left.offset, left.base);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch32(RelationalCondition cond, RegisterID left, RegisterID right)
{
m_assembler.cmpl_rr(right, left);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch32(RelationalCondition cond, RegisterID left, TrustedImm32 right)
{
if (!right.m_value) {
if (auto resultCondition = commuteCompareToZeroIntoTest(cond))
return branchTest32(*resultCondition, left, left);
}
m_assembler.cmpl_ir(right.m_value, left);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch32(RelationalCondition cond, RegisterID left, Address right)
{
m_assembler.cmpl_mr(right.offset, right.base, left);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch32(RelationalCondition cond, Address left, RegisterID right)
{
m_assembler.cmpl_rm(right, left.offset, left.base);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch32(RelationalCondition cond, Address left, TrustedImm32 right)
{
m_assembler.cmpl_im(right.m_value, left.offset, left.base);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch32(RelationalCondition cond, BaseIndex left, TrustedImm32 right)
{
m_assembler.cmpl_im(right.m_value, left.offset, left.base, left.index, left.scale);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch32WithUnalignedHalfWords(RelationalCondition cond, BaseIndex left, TrustedImm32 right)
{
return branch32(cond, left, right);
}
Jump branchTest32(ResultCondition cond, RegisterID reg, RegisterID mask)
{
m_assembler.testl_rr(reg, mask);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchTestBit32(ResultCondition cond, RegisterID reg, TrustedImm32 bit)
{
m_assembler.bt_ir(static_cast<unsigned>(bit.m_value) % 32, reg);
if (cond == NonZero)
return Jump(m_assembler.jb());
if (cond == Zero)
return Jump(m_assembler.jae());
RELEASE_ASSERT_NOT_REACHED();
}
Jump branchTestBit32(ResultCondition cond, Address testValue, TrustedImm32 bit)
{
m_assembler.bt_im(static_cast<unsigned>(bit.m_value) % 32, testValue.offset, testValue.base);
if (cond == NonZero)
return Jump(m_assembler.jb());
if (cond == Zero)
return Jump(m_assembler.jae());
RELEASE_ASSERT_NOT_REACHED();
}
Jump branchTestBit32(ResultCondition cond, RegisterID reg, RegisterID bit)
{
m_assembler.bt_ir(bit, reg);
if (cond == NonZero)
return Jump(m_assembler.jb());
if (cond == Zero)
return Jump(m_assembler.jae());
RELEASE_ASSERT_NOT_REACHED();
}
void test32(RegisterID reg, TrustedImm32 mask = TrustedImm32(-1))
{
if (mask.m_value == -1)
m_assembler.testl_rr(reg, reg);
else if (!(mask.m_value & ~0xff) && reg < X86Registers::esp) { // Using esp and greater as a byte register yields the upper half of the 16 bit registers ax, cx, dx and bx, e.g. esp, register 4, is actually ah.
if (mask.m_value == 0xff)
m_assembler.testb_rr(reg, reg);
else
m_assembler.testb_i8r(mask.m_value, reg);
} else
m_assembler.testl_i32r(mask.m_value, reg);
}
Jump branch(ResultCondition cond)
{
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1))
{
test32(reg, mask);
return branch(cond);
}
Jump branchTest32(ResultCondition cond, Address address, TrustedImm32 mask = TrustedImm32(-1))
{
generateTest32(address, mask);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchTest32(ResultCondition cond, BaseIndex address, TrustedImm32 mask = TrustedImm32(-1))
{
if (mask.m_value == -1)
m_assembler.cmpl_im(0, address.offset, address.base, address.index, address.scale);
else
m_assembler.testl_i32m(mask.m_value, address.offset, address.base, address.index, address.scale);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchTest32(ResultCondition cond, AbsoluteAddress address, TrustedImm32 mask = TrustedImm32(-1))
{
move(TrustedImmPtr(address.m_ptr), scratchRegister());
return branchTest32(cond, Address(scratchRegister()), mask);
}
Jump branchTest8(ResultCondition cond, Address address, TrustedImm32 mask = TrustedImm32(-1))
{
TrustedImm32 mask8(static_cast<int8_t>(mask.m_value));
if (mask8.m_value == -1)
m_assembler.cmpb_im(0, address.offset, address.base);
else
m_assembler.testb_im(mask8.m_value, address.offset, address.base);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchTest8(ResultCondition cond, BaseIndex address, TrustedImm32 mask = TrustedImm32(-1))
{
TrustedImm32 mask8(static_cast<int8_t>(mask.m_value));
if (mask8.m_value == -1)
m_assembler.cmpb_im(0, address.offset, address.base, address.index, address.scale);
else
m_assembler.testb_im(mask8.m_value, address.offset, address.base, address.index, address.scale);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchTest16(ResultCondition cond, Address address, TrustedImm32 mask = TrustedImm32(-1))
{
TrustedImm32 mask16(static_cast<int16_t>(mask.m_value));
if (mask16.m_value == -1)
m_assembler.cmpw_im(0, address.offset, address.base);
else
m_assembler.testw_im(mask16.m_value, address.offset, address.base);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchTest16(ResultCondition cond, BaseIndex address, TrustedImm32 mask = TrustedImm32(-1))
{
TrustedImm32 mask16(static_cast<int16_t>(mask.m_value));
if (mask16.m_value == -1)
m_assembler.cmpw_im(0, address.offset, address.base, address.index, address.scale);
else
m_assembler.testw_im(mask16.m_value, address.offset, address.base, address.index, address.scale);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branch8(RelationalCondition cond, BaseIndex left, TrustedImm32 right)
{
TrustedImm32 right8(static_cast<int8_t>(right.m_value));
m_assembler.cmpb_im(right8.m_value, left.offset, left.base, left.index, left.scale);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump jump()
{
return Jump(m_assembler.jmp());
}
void farJump(RegisterID target, PtrTag)
{
m_assembler.jmp_r(target);
}
void farJump(TrustedImmPtr target, PtrTag)
{
move(target, scratchRegister());
m_assembler.jmp_r(scratchRegister());
}
// Address is a memory location containing the address to jump to
void farJump(Address address, PtrTag)
{
m_assembler.jmp_m(address.offset, address.base);
}
// Address is a memory location containing the address to jump to
void farJump(BaseIndex address, PtrTag)
{
m_assembler.jmp_m(address.offset, address.base, address.index, address.scale);
}
ALWAYS_INLINE void farJump(RegisterID target, RegisterID jumpTag) { UNUSED_PARAM(jumpTag), farJump(target, NoPtrTag); }
ALWAYS_INLINE void farJump(Address address, RegisterID jumpTag) { UNUSED_PARAM(jumpTag), farJump(address, NoPtrTag); }
ALWAYS_INLINE void farJump(BaseIndex address, RegisterID jumpTag) { UNUSED_PARAM(jumpTag), farJump(address, NoPtrTag); }
// Arithmetic control flow operations:
//
// This set of conditional branch operations branch based
// on the result of an arithmetic operation. The operation
// is performed as normal, storing the result.
//
// * jz operations branch if the result is zero.
// * jo operations branch if the (signed) arithmetic
// operation caused an overflow to occur.
Jump branchAdd32(ResultCondition cond, RegisterID src, RegisterID dest)
{
add32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchAdd32(ResultCondition cond, TrustedImm32 imm, RegisterID dest)
{
add32(imm, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchAdd32(ResultCondition cond, TrustedImm32 src, Address dest)
{
add32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchAdd32(ResultCondition cond, RegisterID src, Address dest)
{
add32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchAdd32(ResultCondition cond, Address src, RegisterID dest)
{
add32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchAdd32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest)
{
if (src1 == dest)
return branchAdd32(cond, src2, dest);
move32IfNeeded(src2, dest);
return branchAdd32(cond, src1, dest);
}
Jump branchAdd32(ResultCondition cond, Address op1, RegisterID op2, RegisterID dest)
{
if (op2 == dest)
return branchAdd32(cond, op1, dest);
if (op1.base == dest) {
load32(op1, dest);
return branchAdd32(cond, op2, dest);
}
zeroExtend32ToWord(op2, dest);
return branchAdd32(cond, op1, dest);
}
Jump branchAdd32(ResultCondition cond, RegisterID src1, Address src2, RegisterID dest)
{
return branchAdd32(cond, src2, src1, dest);
}
Jump branchAdd32(ResultCondition cond, RegisterID src, TrustedImm32 imm, RegisterID dest)
{
move32IfNeeded(src, dest);
return branchAdd32(cond, imm, dest);
}
Jump branchMul32(ResultCondition cond, RegisterID src, RegisterID dest)
{
mul32(src, dest);
if (cond != Overflow)
m_assembler.testl_rr(dest, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchMul32(ResultCondition cond, Address src, RegisterID dest)
{
mul32(src, dest);
if (cond != Overflow)
m_assembler.testl_rr(dest, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchMul32(ResultCondition cond, RegisterID src, TrustedImm32 imm, RegisterID dest)
{
mul32(imm, src, dest);
if (cond != Overflow)
m_assembler.testl_rr(dest, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchMul32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest)
{
if (src1 == dest)
return branchMul32(cond, src2, dest);
move32IfNeeded(src2, dest);
return branchMul32(cond, src1, dest);
}
Jump branchSub32(ResultCondition cond, RegisterID src, RegisterID dest)
{
sub32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchSub32(ResultCondition cond, TrustedImm32 imm, RegisterID dest)
{
sub32(imm, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchSub32(ResultCondition cond, TrustedImm32 imm, Address dest)
{
sub32(imm, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchSub32(ResultCondition cond, RegisterID src, Address dest)
{
sub32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchSub32(ResultCondition cond, Address src, RegisterID dest)
{
sub32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchSub32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest)
{
// B := A - B is invalid.
ASSERT(src1 == dest || src2 != dest);
move32IfNeeded(src1, dest);
return branchSub32(cond, src2, dest);
}
Jump branchSub32(ResultCondition cond, RegisterID src1, TrustedImm32 src2, RegisterID dest)
{
move32IfNeeded(src1, dest);
return branchSub32(cond, src2, dest);
}
Jump branchNeg32(ResultCondition cond, RegisterID srcDest)
{
neg32(srcDest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
Jump branchOr32(ResultCondition cond, RegisterID src, RegisterID dest)
{
or32(src, dest);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
// Miscellaneous operations:
void breakpoint()
{
m_assembler.int3();
}
static bool isBreakpoint(void* address) { return X86Assembler::isInt3(address); }
Call nearTailCall()
{
return Call(m_assembler.jmp(), Call::LinkableNearTail);
}
Call nearCall()
{
return Call(m_assembler.call(), Call::LinkableNear);
}
Call call(RegisterID target, PtrTag)
{
return Call(m_assembler.call(target), Call::None);
}
void call(Address address, PtrTag)
{
m_assembler.call_m(address.offset, address.base);
}
ALWAYS_INLINE Call call(RegisterID target, RegisterID callTag) { return UNUSED_PARAM(callTag), call(target, NoPtrTag); }
ALWAYS_INLINE void call(Address address, RegisterID callTag) { UNUSED_PARAM(callTag), call(address, NoPtrTag); }
void ret()
{
m_assembler.ret();
}
void compare8(RelationalCondition cond, Address left, TrustedImm32 right, RegisterID dest)
{
TrustedImm32 right8(static_cast<int8_t>(right.m_value));
m_assembler.cmpb_im(right8.m_value, left.offset, left.base);
set32(x86Condition(cond), dest);
}
void compare32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID dest)
{
m_assembler.cmpl_rr(right, left);
set32(x86Condition(cond), dest);
}
void compare32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID dest)
{
if (!right.m_value) {
if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) {
test32(*resultCondition, left, left, dest);
return;
}
}
m_assembler.cmpl_ir(right.m_value, left);
set32(x86Condition(cond), dest);
}
// FIXME:
// The mask should be optional... perhaps the argument order should be
// dest-src, operations always have a dest? ... possibly not true, considering
// asm ops like test, or pseudo ops like pop().
void test8(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest)
{
TrustedImm32 mask8(static_cast<int8_t>(mask.m_value));
if (mask8.m_value == -1)
m_assembler.cmpb_im(0, address.offset, address.base);
else
m_assembler.testb_im(mask8.m_value, address.offset, address.base);
set32(x86Condition(cond), dest);
}
void test32(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest)
{
generateTest32(address, mask);
set32(x86Condition(cond), dest);
}
void test32(ResultCondition cond, RegisterID reg, RegisterID mask, RegisterID dest)
{
m_assembler.testl_rr(reg, mask);
set32(x86Condition(cond), dest);
}
void test32(ResultCondition cond, RegisterID reg, TrustedImm32 mask, RegisterID dest)
{
test32(reg, mask);
set32(x86Condition(cond), dest);
}
void setCarry(RegisterID dest)
{
set32(X86Assembler::ConditionC, dest);
}
// Invert a relational condition, e.g. == becomes !=, < becomes >=, etc.
static RelationalCondition invert(RelationalCondition cond)
{
return static_cast<RelationalCondition>(cond ^ 1);
}
static DoubleCondition invert(DoubleCondition cond)
{
switch (cond) {
case DoubleEqualAndOrdered:
return DoubleNotEqualOrUnordered;
case DoubleNotEqualAndOrdered:
return DoubleEqualOrUnordered;
case DoubleGreaterThanAndOrdered:
return DoubleLessThanOrEqualOrUnordered;
case DoubleGreaterThanOrEqualAndOrdered:
return DoubleLessThanOrUnordered;
case DoubleLessThanAndOrdered:
return DoubleGreaterThanOrEqualOrUnordered;
case DoubleLessThanOrEqualAndOrdered:
return DoubleGreaterThanOrUnordered;
case DoubleEqualOrUnordered:
return DoubleNotEqualAndOrdered;
case DoubleNotEqualOrUnordered:
return DoubleEqualAndOrdered;
case DoubleGreaterThanOrUnordered:
return DoubleLessThanOrEqualAndOrdered;
case DoubleGreaterThanOrEqualOrUnordered:
return DoubleLessThanAndOrdered;
case DoubleLessThanOrUnordered:
return DoubleGreaterThanOrEqualAndOrdered;
case DoubleLessThanOrEqualOrUnordered:
return DoubleGreaterThanAndOrdered;
}
RELEASE_ASSERT_NOT_REACHED();
return DoubleEqualAndOrdered; // make compiler happy
}
static bool isInvertible(ResultCondition cond)
{
switch (cond) {
case Zero:
case NonZero:
case Signed:
case PositiveOrZero:
return true;
default:
return false;
}
}
static ResultCondition invert(ResultCondition cond)
{
switch (cond) {
case Zero:
return NonZero;
case NonZero:
return Zero;
case Signed:
return PositiveOrZero;
case PositiveOrZero:
return Signed;
default:
RELEASE_ASSERT_NOT_REACHED();
return Zero; // Make compiler happy for release builds.
}
}
static std::optional<ResultCondition> commuteCompareToZeroIntoTest(RelationalCondition cond)
{
switch (cond) {
case Equal:
return Zero;
case NotEqual:
return NonZero;
case LessThan:
return Signed;
case GreaterThanOrEqual:
return PositiveOrZero;
break;
default:
return std::nullopt;
}
}
void nop()
{
m_assembler.nop();
}
void xchg8(RegisterID reg, Address address)
{
m_assembler.xchgb_rm(reg, address.offset, address.base);
}
void xchg8(RegisterID reg, BaseIndex address)
{
m_assembler.xchgb_rm(reg, address.offset, address.base, address.index, address.scale);
}
void xchg16(RegisterID reg, Address address)
{
m_assembler.xchgw_rm(reg, address.offset, address.base);
}
void xchg16(RegisterID reg, BaseIndex address)
{
m_assembler.xchgw_rm(reg, address.offset, address.base, address.index, address.scale);
}
void xchg32(RegisterID reg, Address address)
{
m_assembler.xchgl_rm(reg, address.offset, address.base);
}
void xchg32(RegisterID reg, BaseIndex address)
{
m_assembler.xchgl_rm(reg, address.offset, address.base, address.index, address.scale);
}
// We take memoryFence to mean acqrel. This has acqrel semantics on x86.
void memoryFence()
{
// lock; orl $0, (%rsp)
m_assembler.lock();
m_assembler.orl_im(0, 0, X86Registers::esp);
}
void atomicStrongCAS8(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS(cond, expectedAndResult, result, address, [&] { m_assembler.cmpxchgb_rm(newValue, address.offset, address.base); });
}
void atomicStrongCAS8(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS(cond, expectedAndResult, result, address, [&] { m_assembler.cmpxchgb_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
void atomicStrongCAS16(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS(cond, expectedAndResult, result, address, [&] { m_assembler.cmpxchgw_rm(newValue, address.offset, address.base); });
}
void atomicStrongCAS16(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS(cond, expectedAndResult, result, address, [&] { m_assembler.cmpxchgw_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
void atomicStrongCAS32(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS(cond, expectedAndResult, result, address, [&] { m_assembler.cmpxchgl_rm(newValue, address.offset, address.base); });
}
void atomicStrongCAS32(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS(cond, expectedAndResult, result, address, [&] { m_assembler.cmpxchgl_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
void atomicStrongCAS8(RegisterID expectedAndResult, RegisterID newValue, Address address)
{
atomicStrongCAS(expectedAndResult, address, [&] { m_assembler.cmpxchgb_rm(newValue, address.offset, address.base); });
}
void atomicStrongCAS8(RegisterID expectedAndResult, RegisterID newValue, BaseIndex address)
{
atomicStrongCAS(expectedAndResult, address, [&] { m_assembler.cmpxchgb_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
void atomicStrongCAS16(RegisterID expectedAndResult, RegisterID newValue, Address address)
{
atomicStrongCAS(expectedAndResult, address, [&] { m_assembler.cmpxchgw_rm(newValue, address.offset, address.base); });
}
void atomicStrongCAS16(RegisterID expectedAndResult, RegisterID newValue, BaseIndex address)
{
atomicStrongCAS(expectedAndResult, address, [&] { m_assembler.cmpxchgw_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
void atomicStrongCAS32(RegisterID expectedAndResult, RegisterID newValue, Address address)
{
atomicStrongCAS(expectedAndResult, address, [&] { m_assembler.cmpxchgl_rm(newValue, address.offset, address.base); });
}
void atomicStrongCAS32(RegisterID expectedAndResult, RegisterID newValue, BaseIndex address)
{
atomicStrongCAS(expectedAndResult, address, [&] { m_assembler.cmpxchgl_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
Jump branchAtomicStrongCAS8(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS(cond, expectedAndResult, address, [&] { m_assembler.cmpxchgb_rm(newValue, address.offset, address.base); });
}
Jump branchAtomicStrongCAS8(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS(cond, expectedAndResult, address, [&] { m_assembler.cmpxchgb_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
Jump branchAtomicStrongCAS16(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS(cond, expectedAndResult, address, [&] { m_assembler.cmpxchgw_rm(newValue, address.offset, address.base); });
}
Jump branchAtomicStrongCAS16(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS(cond, expectedAndResult, address, [&] { m_assembler.cmpxchgw_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
Jump branchAtomicStrongCAS32(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS(cond, expectedAndResult, address, [&] { m_assembler.cmpxchgl_rm(newValue, address.offset, address.base); });
}
Jump branchAtomicStrongCAS32(StatusCondition cond, RegisterID expectedAndResult, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS(cond, expectedAndResult, address, [&] { m_assembler.cmpxchgl_rm(newValue, address.offset, address.base, address.index, address.scale); });
}
// If you use weak CAS, you cannot rely on expectedAndClobbered to have any particular value after
// this completes. On x86, it will contain the result of the strong CAS. On ARM, it will still have
// the expected value.
void atomicWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS8(cond, expectedAndClobbered, newValue, address, result);
}
void atomicWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS8(cond, expectedAndClobbered, newValue, address, result);
}
void atomicWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS16(cond, expectedAndClobbered, newValue, address, result);
}
void atomicWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS16(cond, expectedAndClobbered, newValue, address, result);
}
void atomicWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS32(cond, expectedAndClobbered, newValue, address, result);
}
void atomicWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS32(cond, expectedAndClobbered, newValue, address, result);
}
Jump branchAtomicWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS8(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS8(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS16(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS16(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS32(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS32(cond, expectedAndClobbered, newValue, address);
}
void atomicRelaxedWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS8(cond, expectedAndClobbered, newValue, address, result);
}
void atomicRelaxedWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS8(cond, expectedAndClobbered, newValue, address, result);
}
void atomicRelaxedWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS16(cond, expectedAndClobbered, newValue, address, result);
}
void atomicRelaxedWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS16(cond, expectedAndClobbered, newValue, address, result);
}
void atomicRelaxedWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address, RegisterID result)
{
atomicStrongCAS32(cond, expectedAndClobbered, newValue, address, result);
}
void atomicRelaxedWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address, RegisterID result)
{
atomicStrongCAS32(cond, expectedAndClobbered, newValue, address, result);
}
Jump branchAtomicRelaxedWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS8(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicRelaxedWeakCAS8(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS8(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicRelaxedWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS16(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicRelaxedWeakCAS16(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS16(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicRelaxedWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, Address address)
{
return branchAtomicStrongCAS32(cond, expectedAndClobbered, newValue, address);
}
Jump branchAtomicRelaxedWeakCAS32(StatusCondition cond, RegisterID expectedAndClobbered, RegisterID newValue, BaseIndex address)
{
return branchAtomicStrongCAS32(cond, expectedAndClobbered, newValue, address);
}
void atomicAdd8(TrustedImm32 imm, Address address)
{
m_assembler.lock();
add8(imm, address);
}
void atomicAdd8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
add8(imm, address);
}
void atomicAdd8(RegisterID reg, Address address)
{
m_assembler.lock();
add8(reg, address);
}
void atomicAdd8(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
add8(reg, address);
}
void atomicAdd16(TrustedImm32 imm, Address address)
{
m_assembler.lock();
add16(imm, address);
}
void atomicAdd16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
add16(imm, address);
}
void atomicAdd16(RegisterID reg, Address address)
{
m_assembler.lock();
add16(reg, address);
}
void atomicAdd16(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
add16(reg, address);
}
void atomicAdd32(TrustedImm32 imm, Address address)
{
m_assembler.lock();
add32(imm, address);
}
void atomicAdd32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
add32(imm, address);
}
void atomicAdd32(RegisterID reg, Address address)
{
m_assembler.lock();
add32(reg, address);
}
void atomicAdd32(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
add32(reg, address);
}
void atomicSub8(TrustedImm32 imm, Address address)
{
m_assembler.lock();
sub8(imm, address);
}
void atomicSub8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
sub8(imm, address);
}
void atomicSub8(RegisterID reg, Address address)
{
m_assembler.lock();
sub8(reg, address);
}
void atomicSub8(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
sub8(reg, address);
}
void atomicSub16(TrustedImm32 imm, Address address)
{
m_assembler.lock();
sub16(imm, address);
}
void atomicSub16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
sub16(imm, address);
}
void atomicSub16(RegisterID reg, Address address)
{
m_assembler.lock();
sub16(reg, address);
}
void atomicSub16(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
sub16(reg, address);
}
void atomicSub32(TrustedImm32 imm, Address address)
{
m_assembler.lock();
sub32(imm, address);
}
void atomicSub32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
sub32(imm, address);
}
void atomicSub32(RegisterID reg, Address address)
{
m_assembler.lock();
sub32(reg, address);
}
void atomicSub32(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
sub32(reg, address);
}
void atomicAnd8(TrustedImm32 imm, Address address)
{
m_assembler.lock();
and8(imm, address);
}
void atomicAnd8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
and8(imm, address);
}
void atomicAnd8(RegisterID reg, Address address)
{
m_assembler.lock();
and8(reg, address);
}
void atomicAnd8(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
and8(reg, address);
}
void atomicAnd16(TrustedImm32 imm, Address address)
{
m_assembler.lock();
and16(imm, address);
}
void atomicAnd16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
and16(imm, address);
}
void atomicAnd16(RegisterID reg, Address address)
{
m_assembler.lock();
and16(reg, address);
}
void atomicAnd16(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
and16(reg, address);
}
void atomicAnd32(TrustedImm32 imm, Address address)
{
m_assembler.lock();
and32(imm, address);
}
void atomicAnd32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
and32(imm, address);
}
void atomicAnd32(RegisterID reg, Address address)
{
m_assembler.lock();
and32(reg, address);
}
void atomicAnd32(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
and32(reg, address);
}
void atomicOr8(TrustedImm32 imm, Address address)
{
m_assembler.lock();
or8(imm, address);
}
void atomicOr8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
or8(imm, address);
}
void atomicOr8(RegisterID reg, Address address)
{
m_assembler.lock();
or8(reg, address);
}
void atomicOr8(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
or8(reg, address);
}
void atomicOr16(TrustedImm32 imm, Address address)
{
m_assembler.lock();
or16(imm, address);
}
void atomicOr16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
or16(imm, address);
}
void atomicOr16(RegisterID reg, Address address)
{
m_assembler.lock();
or16(reg, address);
}
void atomicOr16(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
or16(reg, address);
}
void atomicOr32(TrustedImm32 imm, Address address)
{
m_assembler.lock();
or32(imm, address);
}
void atomicOr32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
or32(imm, address);
}
void atomicOr32(RegisterID reg, Address address)
{
m_assembler.lock();
or32(reg, address);
}
void atomicOr32(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
or32(reg, address);
}
void atomicXor8(TrustedImm32 imm, Address address)
{
m_assembler.lock();
xor8(imm, address);
}
void atomicXor8(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
xor8(imm, address);
}
void atomicXor8(RegisterID reg, Address address)
{
m_assembler.lock();
xor8(reg, address);
}
void atomicXor8(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
xor8(reg, address);
}
void atomicXor16(TrustedImm32 imm, Address address)
{
m_assembler.lock();
xor16(imm, address);
}
void atomicXor16(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
xor16(imm, address);
}
void atomicXor16(RegisterID reg, Address address)
{
m_assembler.lock();
xor16(reg, address);
}
void atomicXor16(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
xor16(reg, address);
}
void atomicXor32(TrustedImm32 imm, Address address)
{
m_assembler.lock();
xor32(imm, address);
}
void atomicXor32(TrustedImm32 imm, BaseIndex address)
{
m_assembler.lock();
xor32(imm, address);
}
void atomicXor32(RegisterID reg, Address address)
{
m_assembler.lock();
xor32(reg, address);
}
void atomicXor32(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
xor32(reg, address);
}
void atomicNeg8(Address address)
{
m_assembler.lock();
neg8(address);
}
void atomicNeg8(BaseIndex address)
{
m_assembler.lock();
neg8(address);
}
void atomicNeg16(Address address)
{
m_assembler.lock();
neg16(address);
}
void atomicNeg16(BaseIndex address)
{
m_assembler.lock();
neg16(address);
}
void atomicNeg32(Address address)
{
m_assembler.lock();
neg32(address);
}
void atomicNeg32(BaseIndex address)
{
m_assembler.lock();
neg32(address);
}
void atomicNot8(Address address)
{
m_assembler.lock();
not8(address);
}
void atomicNot8(BaseIndex address)
{
m_assembler.lock();
not8(address);
}
void atomicNot16(Address address)
{
m_assembler.lock();
not16(address);
}
void atomicNot16(BaseIndex address)
{
m_assembler.lock();
not16(address);
}
void atomicNot32(Address address)
{
m_assembler.lock();
not32(address);
}
void atomicNot32(BaseIndex address)
{
m_assembler.lock();
not32(address);
}
void atomicXchgAdd8(RegisterID reg, Address address)
{
m_assembler.lock();
m_assembler.xaddb_rm(reg, address.offset, address.base);
}
void atomicXchgAdd8(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
m_assembler.xaddb_rm(reg, address.offset, address.base, address.index, address.scale);
}
void atomicXchgAdd16(RegisterID reg, Address address)
{
m_assembler.lock();
m_assembler.xaddw_rm(reg, address.offset, address.base);
}
void atomicXchgAdd16(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
m_assembler.xaddw_rm(reg, address.offset, address.base, address.index, address.scale);
}
void atomicXchgAdd32(RegisterID reg, Address address)
{
m_assembler.lock();
m_assembler.xaddl_rm(reg, address.offset, address.base);
}
void atomicXchgAdd32(RegisterID reg, BaseIndex address)
{
m_assembler.lock();
m_assembler.xaddl_rm(reg, address.offset, address.base, address.index, address.scale);
}
void atomicXchg8(RegisterID reg, Address address)
{
m_assembler.xchgb_rm(reg, address.offset, address.base);
}
void atomicXchg8(RegisterID reg, BaseIndex address)
{
m_assembler.xchgb_rm(reg, address.offset, address.base, address.index, address.scale);
}
void atomicXchg16(RegisterID reg, Address address)
{
m_assembler.xchgw_rm(reg, address.offset, address.base);
}
void atomicXchg16(RegisterID reg, BaseIndex address)
{
m_assembler.xchgw_rm(reg, address.offset, address.base, address.index, address.scale);
}
void atomicXchg32(RegisterID reg, Address address)
{
m_assembler.xchgl_rm(reg, address.offset, address.base);
}
void atomicXchg32(RegisterID reg, BaseIndex address)
{
m_assembler.xchgl_rm(reg, address.offset, address.base, address.index, address.scale);
}
// We take this to mean that it prevents motion of normal stores. So, it's a no-op on x86.
void storeFence()
{
}
// We take this to mean that it prevents motion of normal loads. So, it's a no-op on x86.
void loadFence()
{
}
#if ENABLE(FAST_TLS_JIT)
void loadFromTLS32(uint32_t offset, RegisterID dst)
{
m_assembler.gs();
m_assembler.movl_mr(offset, dst);
}
static bool loadFromTLSPtrNeedsMacroScratchRegister()
{
return false;
}
void storeToTLS32(RegisterID src, uint32_t offset)
{
m_assembler.gs();
m_assembler.movl_rm(src, offset);
}
static bool storeToTLSPtrNeedsMacroScratchRegister()
{
return false;
}
#endif
template<PtrTag tag>
static void replaceWithVMHalt(CodeLocationLabel<tag> instructionStart)
{
X86Assembler::replaceWithHlt(instructionStart.executableAddress());
}
template<PtrTag startTag, PtrTag destTag>
static void replaceWithJump(CodeLocationLabel<startTag> instructionStart, CodeLocationLabel<destTag> destination)
{
X86Assembler::replaceWithJump(instructionStart.executableAddress(), destination.executableAddress());
}
static ptrdiff_t maxJumpReplacementSize()
{
return X86Assembler::maxJumpReplacementSize();
}
static ptrdiff_t patchableJumpSize()
{
return X86Assembler::patchableJumpSize();
}
static bool supportsFloatingPointRounding()
{
if (s_sse4_1CheckState == CPUIDCheckState::NotChecked)
collectCPUFeatures();
return s_sse4_1CheckState == CPUIDCheckState::Set;
}
static bool supportsCountPopulation()
{
if (s_popcntCheckState == CPUIDCheckState::NotChecked)
collectCPUFeatures();
return s_popcntCheckState == CPUIDCheckState::Set;
}
static bool supportsAVX()
{
// AVX still causes mysterious regressions and those regressions can be massive.
return false;
}
void lfence()
{
m_assembler.lfence();
}
void mfence()
{
m_assembler.mfence();
}
void sfence()
{
m_assembler.sfence();
}
void rdtsc()
{
m_assembler.rdtsc();
}
void pause()
{
m_assembler.pause();
}
void cpuid()
{
m_assembler.cpuid();
}
protected:
X86Assembler::Condition x86Condition(RelationalCondition cond)
{
return static_cast<X86Assembler::Condition>(cond);
}
X86Assembler::Condition x86Condition(ResultCondition cond)
{
return static_cast<X86Assembler::Condition>(cond);
}
X86Assembler::Condition x86Condition(StatusCondition cond)
{
switch (cond) {
case Success:
return X86Assembler::ConditionE;
case Failure:
return X86Assembler::ConditionNE;
}
RELEASE_ASSERT_NOT_REACHED();
return X86Assembler::ConditionE;
}
void set32(X86Assembler::Condition cond, RegisterID dest)
{
#if CPU(X86)
// On 32-bit x86 we can only set the first 4 registers;
// esp..edi are mapped to the 'h' registers!
if (dest >= 4) {
m_assembler.xchgl_rr(dest, X86Registers::eax);
m_assembler.setCC_r(cond, X86Registers::eax);
m_assembler.movzbl_rr(X86Registers::eax, X86Registers::eax);
m_assembler.xchgl_rr(dest, X86Registers::eax);
return;
}
#endif
m_assembler.setCC_r(cond, dest);
m_assembler.movzbl_rr(dest, dest);
}
void cmov(X86Assembler::Condition cond, RegisterID src, RegisterID dest)
{
#if CPU(X86_64)
m_assembler.cmovq_rr(cond, src, dest);
#else
m_assembler.cmovl_rr(cond, src, dest);
#endif
}
static bool supportsLZCNT()
{
if (s_lzcntCheckState == CPUIDCheckState::NotChecked)
collectCPUFeatures();
return s_lzcntCheckState == CPUIDCheckState::Set;
}
static bool supportsBMI1()
{
if (s_bmi1CheckState == CPUIDCheckState::NotChecked)
collectCPUFeatures();
return s_bmi1CheckState == CPUIDCheckState::Set;
}
template<int sizeOfRegister>
void ctzAfterBsf(RegisterID dst)
{
Jump srcIsNonZero = m_assembler.jCC(x86Condition(NonZero));
move(TrustedImm32(sizeOfRegister), dst);
srcIsNonZero.link(this);
}
template<typename AddressType, typename Func>
void atomicStrongCAS(StatusCondition cond, RegisterID expectedAndResult, RegisterID result, AddressType& address, const Func& func)
{
address = address.withSwappedRegister(X86Registers::eax, expectedAndResult);
swap(expectedAndResult, X86Registers::eax);
m_assembler.lock();
func();
swap(expectedAndResult, X86Registers::eax);
set32(x86Condition(cond), result);
}
template<typename AddressType, typename Func>
void atomicStrongCAS(RegisterID expectedAndResult, AddressType& address, const Func& func)
{
address = address.withSwappedRegister(X86Registers::eax, expectedAndResult);
swap(expectedAndResult, X86Registers::eax);
m_assembler.lock();
func();
swap(expectedAndResult, X86Registers::eax);
}
template<typename AddressType, typename Func>
Jump branchAtomicStrongCAS(StatusCondition cond, RegisterID expectedAndResult, AddressType& address, const Func& func)
{
address = address.withSwappedRegister(X86Registers::eax, expectedAndResult);
swap(expectedAndResult, X86Registers::eax);
m_assembler.lock();
func();
swap(expectedAndResult, X86Registers::eax);
return Jump(m_assembler.jCC(x86Condition(cond)));
}
private:
// Only MacroAssemblerX86 should be using the following method; SSE2 is always available on
// x86_64, and clients & subclasses of MacroAssembler should be using 'supportsFloatingPoint()'.
friend class MacroAssemblerX86;
ALWAYS_INLINE void generateTest32(Address address, TrustedImm32 mask = TrustedImm32(-1))
{
if (mask.m_value == -1)
m_assembler.cmpl_im(0, address.offset, address.base);
else if (!(mask.m_value & ~0xff))
m_assembler.testb_im(mask.m_value, address.offset, address.base);
else if (!(mask.m_value & ~0xff00))
m_assembler.testb_im(mask.m_value >> 8, address.offset + 1, address.base);
else if (!(mask.m_value & ~0xff0000))
m_assembler.testb_im(mask.m_value >> 16, address.offset + 2, address.base);
else if (!(mask.m_value & ~0xff000000))
m_assembler.testb_im(mask.m_value >> 24, address.offset + 3, address.base);
else
m_assembler.testl_i32m(mask.m_value, address.offset, address.base);
}
// If lzcnt is not available, use this after BSR
// to count the leading zeros.
void clz32AfterBsr(RegisterID dst)
{
Jump srcIsNonZero = m_assembler.jCC(x86Condition(NonZero));
move(TrustedImm32(32), dst);
Jump skipNonZeroCase = jump();
srcIsNonZero.link(this);
xor32(TrustedImm32(0x1f), dst);
skipNonZeroCase.link(this);
}
template<typename Function>
void floatingPointCompare(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID dest, Function compare)
{
if (cond & DoubleConditionBitSpecial) {
ASSERT(!(cond & DoubleConditionBitInvert));
if (cond == DoubleEqualAndOrdered) {
if (left == right) {
compare(right, left);
set32(X86Assembler::ConditionNP, dest);
return;
}
move(TrustedImm32(0), dest);
compare(right, left);
Jump isUnordered = m_assembler.jp();
set32(X86Assembler::ConditionE, dest);
isUnordered.link(this);
return;
}
if (cond == DoubleNotEqualOrUnordered) {
if (left == right) {
compare(right, left);
set32(X86Assembler::ConditionP, dest);
return;
}
move(TrustedImm32(1), dest);
compare(right, left);
Jump isUnordered = m_assembler.jp();
set32(X86Assembler::ConditionNE, dest);
isUnordered.link(this);
return;
}
RELEASE_ASSERT_NOT_REACHED();
return;
}
if (cond & DoubleConditionBitInvert)
compare(left, right);
else
compare(right, left);
set32(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits), dest);
}
Jump jumpAfterFloatingPointCompare(DoubleCondition cond, FPRegisterID left, FPRegisterID right)
{
if (cond == DoubleEqualAndOrdered) {
if (left == right)
return Jump(m_assembler.jnp());
Jump isUnordered(m_assembler.jp());
Jump result = Jump(m_assembler.je());
isUnordered.link(this);
return result;
}
if (cond == DoubleNotEqualOrUnordered) {
if (left == right)
return Jump(m_assembler.jp());
Jump isUnordered(m_assembler.jp());
Jump isEqual(m_assembler.je());
isUnordered.link(this);
Jump result = jump();
isEqual.link(this);
return result;
}
ASSERT(!(cond & DoubleConditionBitSpecial));
return Jump(m_assembler.jCC(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits)));
}
// The 32bit Move does not need the REX byte for low registers, making it shorter.
// Use this if the top bits are irrelevant because they will be reset by the next instruction.
void move32IfNeeded(RegisterID src, RegisterID dest)
{
if (src == dest)
return;
m_assembler.movl_rr(src, dest);
}
#if CPU(X86_64)
void moveConditionallyAfterFloatingPointCompare(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest)
{
if (cond == DoubleEqualAndOrdered) {
if (left == right) {
m_assembler.cmovnpq_rr(src, dest);
return;
}
Jump isUnordered(m_assembler.jp());
m_assembler.cmoveq_rr(src, dest);
isUnordered.link(this);
return;
}
if (cond == DoubleNotEqualOrUnordered) {
if (left == right) {
m_assembler.cmovpq_rr(src, dest);
return;
}
m_assembler.cmovpq_rr(src, dest);
m_assembler.cmovneq_rr(src, dest);
return;
}
ASSERT(!(cond & DoubleConditionBitSpecial));
cmov(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits), src, dest);
}
#endif
using CPUID = std::array<unsigned, 4>;
static CPUID getCPUID(unsigned level);
static CPUID getCPUIDEx(unsigned level, unsigned count);
JS_EXPORT_PRIVATE static void collectCPUFeatures();
JS_EXPORT_PRIVATE static CPUIDCheckState s_sse4_1CheckState;
JS_EXPORT_PRIVATE static CPUIDCheckState s_sse4_2CheckState;
JS_EXPORT_PRIVATE static CPUIDCheckState s_avxCheckState;
JS_EXPORT_PRIVATE static CPUIDCheckState s_lzcntCheckState;
JS_EXPORT_PRIVATE static CPUIDCheckState s_bmi1CheckState;
JS_EXPORT_PRIVATE static CPUIDCheckState s_popcntCheckState;
};
} // namespace JSC
#endif // ENABLE(ASSEMBLER)