blob: 7831ef36690d70bd5e50412bbabda85c4b0905cc [file] [log] [blame]
/*
* Copyright (C) 2021 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#if ENABLE(JIT)
#include "AssemblyHelpers.h"
namespace JSC {
template<typename RegType>
struct RegDispatch {
static bool hasSameType(Reg);
static RegType get(Reg);
template<typename Spooler> static RegType temp1(const Spooler*);
template<typename Spooler> static RegType temp2(const Spooler*);
template<typename Spooler> static RegType& regToStore(Spooler*);
static constexpr RegType invalid();
static constexpr size_t regSize();
static bool isValidLoadPairImm(int);
static bool isValidStorePairImm(int);
};
template<>
struct RegDispatch<GPRReg> {
static bool hasSameType(Reg reg) { return reg.isGPR(); }
static GPRReg get(Reg reg) { return reg.gpr(); }
template<typename Spooler> static GPRReg temp1(const Spooler* spooler) { return spooler->m_temp1GPR; }
template<typename Spooler> static GPRReg temp2(const Spooler* spooler) { return spooler->m_temp2GPR; }
template<typename Spooler> static GPRReg& regToStore(Spooler* spooler) { return spooler->m_gprToStore; }
static constexpr GPRReg invalid() { return InvalidGPRReg; }
static constexpr size_t regSize() { return sizeof(CPURegister); }
#if CPU(ARM64)
static bool isValidLoadPairImm(int offset) { return ARM64Assembler::isValidLDPImm<64>(offset); }
static bool isValidStorePairImm(int offset) { return ARM64Assembler::isValidSTPImm<64>(offset); }
#else
static bool isValidLoadPairImm(int) { return false; }
static bool isValidStorePairImm(int) { return false; }
#endif
};
template<>
struct RegDispatch<FPRReg> {
static bool hasSameType(Reg reg) { return reg.isFPR(); }
static FPRReg get(Reg reg) { return reg.fpr(); }
template<typename Spooler> static FPRReg temp1(const Spooler* spooler) { return spooler->m_temp1FPR; }
template<typename Spooler> static FPRReg temp2(const Spooler* spooler) { return spooler->m_temp2FPR; }
template<typename Spooler> static FPRReg& regToStore(Spooler* spooler) { return spooler->m_fprToStore; }
static constexpr FPRReg invalid() { return InvalidFPRReg; }
static constexpr size_t regSize() { return sizeof(double); }
#if CPU(ARM64)
static bool isValidLoadPairImm(int offset) { return ARM64Assembler::isValidLDPFPImm<64>(offset); }
static bool isValidStorePairImm(int offset) { return ARM64Assembler::isValidSTPFPImm<64>(offset); }
#else
static bool isValidLoadPairImm(int) { return false; }
static bool isValidStorePairImm(int) { return false; }
#endif
};
template<typename Op>
class AssemblyHelpers::Spooler {
public:
using JIT = AssemblyHelpers;
Spooler(JIT& jit, GPRReg baseGPR)
: m_jit(jit)
, m_baseGPR(baseGPR)
{ }
template<typename RegType>
void execute(const RegisterAtOffset& entry)
{
RELEASE_ASSERT(RegDispatch<RegType>::hasSameType(entry.reg()));
if constexpr (!hasPairOp)
return op().executeSingle(entry.offset(), RegDispatch<RegType>::get(entry.reg()));
if (!m_bufferedEntry.reg().isSet()) {
m_bufferedEntry = entry;
return;
}
constexpr ptrdiff_t regSize = RegDispatch<RegType>::regSize();
RegType bufferedEntryReg = RegDispatch<RegType>::get(m_bufferedEntry.reg());
RegType entryReg = RegDispatch<RegType>::get(entry.reg());
if (entry.offset() == m_bufferedEntry.offset() + regSize) {
op().executePair(m_bufferedEntry.offset(), bufferedEntryReg, entryReg);
m_bufferedEntry = { };
return;
}
if (m_bufferedEntry.offset() == entry.offset() + regSize) {
op().executePair(entry.offset(), entryReg, bufferedEntryReg);
m_bufferedEntry = { };
return;
}
// We don't have a pair of operations that we can execute as a pair.
// Execute the previous one as a single (finalize will do that), and then
// buffer the current entry to potentially be paired with the next entry.
finalize<RegType>();
execute<RegType>(entry);
}
template<typename RegType>
void finalize()
{
if constexpr (hasPairOp) {
if (m_bufferedEntry.reg().isSet()) {
op().executeSingle(m_bufferedEntry.offset(), RegDispatch<RegType>::get(m_bufferedEntry.reg()));
m_bufferedEntry = { };
}
}
}
private:
#if CPU(ARM64)
static constexpr bool hasPairOp = true;
#else
static constexpr bool hasPairOp = false;
#endif
Op& op() { return *reinterpret_cast<Op*>(this); }
protected:
JIT& m_jit;
GPRReg m_baseGPR;
RegisterAtOffset m_bufferedEntry;
};
class AssemblyHelpers::LoadRegSpooler : public AssemblyHelpers::Spooler<LoadRegSpooler> {
using Base = Spooler<LoadRegSpooler>;
using JIT = Base::JIT;
public:
LoadRegSpooler(JIT& jit, GPRReg baseGPR)
: Base(jit, baseGPR)
{ }
ALWAYS_INLINE void loadGPR(const RegisterAtOffset& entry) { execute<GPRReg>(entry); }
ALWAYS_INLINE void finalizeGPR() { finalize<GPRReg>(); }
ALWAYS_INLINE void loadFPR(const RegisterAtOffset& entry) { execute<FPRReg>(entry); }
ALWAYS_INLINE void finalizeFPR() { finalize<FPRReg>(); }
private:
#if CPU(ARM64)
template<typename RegType>
ALWAYS_INLINE void executePair(ptrdiff_t offset, RegType reg1, RegType reg2)
{
m_jit.loadPair64(m_baseGPR, TrustedImm32(offset), reg1, reg2);
}
#else
template<typename RegType>
ALWAYS_INLINE void executePair(ptrdiff_t, RegType, RegType) { }
#endif
ALWAYS_INLINE void executeSingle(ptrdiff_t offset, GPRReg reg)
{
#if USE(JSVALUE64)
m_jit.load64(Address(m_baseGPR, offset), reg);
#else
m_jit.load32(Address(m_baseGPR, offset), reg);
#endif
}
ALWAYS_INLINE void executeSingle(ptrdiff_t offset, FPRReg reg)
{
m_jit.loadDouble(Address(m_baseGPR, offset), reg);
}
friend class AssemblyHelpers::Spooler<LoadRegSpooler>;
};
class AssemblyHelpers::StoreRegSpooler : public AssemblyHelpers::Spooler<StoreRegSpooler> {
using Base = Spooler<StoreRegSpooler>;
using JIT = typename Base::JIT;
public:
StoreRegSpooler(JIT& jit, GPRReg baseGPR)
: Base(jit, baseGPR)
{ }
ALWAYS_INLINE void storeGPR(const RegisterAtOffset& entry) { execute<GPRReg>(entry); }
ALWAYS_INLINE void finalizeGPR() { finalize<GPRReg>(); }
ALWAYS_INLINE void storeFPR(const RegisterAtOffset& entry) { execute<FPRReg>(entry); }
ALWAYS_INLINE void finalizeFPR() { finalize<FPRReg>(); }
private:
#if CPU(ARM64)
template<typename RegType>
ALWAYS_INLINE void executePair(ptrdiff_t offset, RegType reg1, RegType reg2)
{
m_jit.storePair64(reg1, reg2, m_baseGPR, TrustedImm32(offset));
}
#else
template<typename RegType>
ALWAYS_INLINE void executePair(ptrdiff_t, RegType, RegType) { }
#endif
ALWAYS_INLINE void executeSingle(ptrdiff_t offset, GPRReg reg)
{
#if USE(JSVALUE64)
m_jit.store64(reg, Address(m_baseGPR, offset));
#else
m_jit.store32(reg, Address(m_baseGPR, offset));
#endif
}
ALWAYS_INLINE void executeSingle(ptrdiff_t offset, FPRReg reg)
{
m_jit.storeDouble(reg, Address(m_baseGPR, offset));
}
friend class AssemblyHelpers::Spooler<StoreRegSpooler>;
};
class AssemblyHelpers::CopySpooler {
public:
using JIT = AssemblyHelpers;
using Address = JIT::Address;
using TrustedImm32 = JIT::TrustedImm32;
struct Source {
enum class Type { BufferOffset, Reg, EncodedJSValue } type;
int offset;
Reg reg;
EncodedJSValue value;
template<typename RegType> RegType getReg() { return RegDispatch<RegType>::get(reg); };
};
enum class BufferRegs {
NeedPreservation,
AllowModification
};
CopySpooler(BufferRegs attribute, JIT& jit, GPRReg srcBuffer, GPRReg destBuffer, GPRReg temp1, GPRReg temp2, FPRReg fpTemp1 = InvalidFPRReg, FPRReg fpTemp2 = InvalidFPRReg)
: m_jit(jit)
, m_srcBufferGPR(srcBuffer)
, m_dstBufferGPR(destBuffer)
, m_temp1GPR(temp1)
, m_temp2GPR(temp2)
, m_temp1FPR(fpTemp1)
, m_temp2FPR(fpTemp2)
, m_bufferRegsAttr(attribute)
{
if constexpr (hasPairOp && !isARM64())
RELEASE_ASSERT_NOT_REACHED(); // unsupported architecture.
}
CopySpooler(JIT& jit, GPRReg srcBuffer, GPRReg destBuffer, GPRReg temp1, GPRReg temp2, FPRReg fpTemp1 = InvalidFPRReg, FPRReg fpTemp2 = InvalidFPRReg)
: CopySpooler(BufferRegs::NeedPreservation, jit, srcBuffer, destBuffer, temp1, temp2, fpTemp1, fpTemp2)
{ }
private:
template<typename RegType> RegType temp1() const { return RegDispatch<RegType>::temp1(this); }
template<typename RegType> RegType temp2() const { return RegDispatch<RegType>::temp2(this); }
template<typename RegType> RegType& regToStore() { return RegDispatch<RegType>::regToStore(this); }
template<typename RegType> static constexpr RegType invalid() { return RegDispatch<RegType>::invalid(); }
template<typename RegType> static constexpr int regSize() { return RegDispatch<RegType>::regSize(); }
template<typename RegType> static bool isValidLoadPairImm(int offset) { return RegDispatch<RegType>::isValidLoadPairImm(offset); }
template<typename RegType> static bool isValidStorePairImm(int offset) { return RegDispatch<RegType>::isValidStorePairImm(offset); }
template<typename RegType>
void load(int offset)
{
if constexpr (!hasPairOp) {
auto& regToStore = this->regToStore<RegType>();
regToStore = temp1<RegType>();
load(offset, regToStore);
return;
}
auto& source = m_sources[m_currentSource++];
source.type = Source::Type::BufferOffset;
source.offset = offset;
}
void move(EncodedJSValue value)
{
if constexpr (!hasPairOp) {
auto& regToStore = this->regToStore<GPRReg>();
regToStore = temp1<GPRReg>();
move(value, regToStore);
return;
}
auto& source = m_sources[m_currentSource++];
source.type = Source::Type::EncodedJSValue;
source.value = value;
}
template<typename RegType>
void copy(RegType reg)
{
if constexpr (!hasPairOp) {
auto& regToStore = this->regToStore<RegType>();
regToStore = reg;
return;
}
auto& source = m_sources[m_currentSource++];
source.type = Source::Type::Reg;
source.reg = reg;
}
template<typename RegType>
void store(int storeOffset)
{
if constexpr (!hasPairOp) {
auto regToStore = this->regToStore<RegType>();
store(regToStore, storeOffset);
return;
}
constexpr bool regTypeIsGPR = std::is_same<RegType, GPRReg>::value;
if (m_currentSource < 2) {
m_deferredStoreOffset = storeOffset;
return;
}
RegType regToStore1 = invalid<RegType>();
RegType regToStore2 = invalid<RegType>();
auto& source1 = m_sources[0];
auto& source2 = m_sources[1];
auto srcOffset1 = m_sources[0].offset - m_srcOffsetAdjustment;
auto srcOffset2 = m_sources[1].offset - m_srcOffsetAdjustment;
constexpr int registerSize = regSize<RegType>();
if (source1.type == Source::Type::BufferOffset && source2.type == Source::Type::BufferOffset) {
regToStore1 = temp1<RegType>();
regToStore2 = temp2<RegType>();
int offsetDelta = abs(srcOffset1 - srcOffset2);
int minOffset = std::min(srcOffset1, srcOffset2);
bool isValidOffset = isValidLoadPairImm<RegType>(minOffset);
if (offsetDelta != registerSize || (!isValidOffset && m_bufferRegsAttr != BufferRegs::AllowModification)) {
load(srcOffset1, regToStore1);
load(srcOffset2, regToStore2);
} else {
if (!isValidOffset) {
ASSERT(m_bufferRegsAttr == BufferRegs::AllowModification);
m_srcOffsetAdjustment += minOffset;
m_jit.addPtr(TrustedImm32(minOffset), m_srcBufferGPR);
srcOffset1 -= minOffset;
srcOffset2 -= minOffset;
ASSERT(isValidLoadPairImm<RegType>(std::min(srcOffset1, srcOffset2)));
}
if (srcOffset1 < srcOffset2)
loadPair(srcOffset1, regToStore1, regToStore2);
else
loadPair(srcOffset2, regToStore2, regToStore1);
}
} else if (source1.type == Source::Type::BufferOffset) {
regToStore1 = temp1<RegType>();
load(srcOffset1, regToStore1);
if (source2.type == Source::Type::EncodedJSValue) {
if constexpr (regTypeIsGPR) {
regToStore2 = temp2<RegType>();
move(source2.value, regToStore2);
} else
RELEASE_ASSERT_NOT_REACHED();
} else
regToStore2 = source2.getReg<RegType>();
} else if (source2.type == Source::Type::BufferOffset) {
if (source1.type == Source::Type::EncodedJSValue) {
if constexpr (regTypeIsGPR) {
regToStore1 = temp1<RegType>();
move(source1.value, regToStore1);
} else
RELEASE_ASSERT_NOT_REACHED();
} else
regToStore1 = source1.getReg<RegType>();
regToStore2 = temp2<RegType>();
load(srcOffset2, regToStore2);
} else {
if (source1.type == Source::Type::EncodedJSValue) {
if constexpr (regTypeIsGPR) {
regToStore1 = temp1<RegType>();
move(source1.value, regToStore1);
} else
RELEASE_ASSERT_NOT_REACHED();
} else
regToStore1 = source1.getReg<RegType>();
if (source2.type == Source::Type::EncodedJSValue) {
if constexpr (regTypeIsGPR) {
regToStore2 = temp2<RegType>();
move(source2.value, regToStore2);
} else
RELEASE_ASSERT_NOT_REACHED();
} else
regToStore2 = source2.getReg<RegType>();
}
int dstOffset1 = m_deferredStoreOffset - m_dstOffsetAdjustment;
int dstOffset2 = storeOffset - m_dstOffsetAdjustment;
int offsetDelta = abs(dstOffset1 - dstOffset2);
int minOffset = std::min(dstOffset1, dstOffset2);
bool isValidOffset = isValidStorePairImm<RegType>(minOffset);
if (offsetDelta != registerSize || (!isValidOffset && m_bufferRegsAttr != BufferRegs::AllowModification)) {
store(regToStore1, dstOffset1);
store(regToStore2, dstOffset2);
} else {
if (!isValidOffset) {
ASSERT(m_bufferRegsAttr == BufferRegs::AllowModification);
m_dstOffsetAdjustment += minOffset;
m_jit.addPtr(TrustedImm32(minOffset), m_dstBufferGPR);
dstOffset1 -= minOffset;
dstOffset2 -= minOffset;
ASSERT(isValidStorePairImm<RegType>(std::min(dstOffset1, dstOffset2)));
}
if (dstOffset1 < dstOffset2)
storePair(regToStore1, regToStore2, dstOffset1);
else
storePair(regToStore2, regToStore1, dstOffset2);
}
m_currentSource = 0;
}
template<typename RegType>
void finalize()
{
if constexpr (!hasPairOp)
return;
if (!m_currentSource)
return; // Nothing to finalize.
ASSERT(m_currentSource == 1);
RegType regToStore = invalid<RegType>();
auto& source = m_sources[0];
auto& srcOffset = source.offset;
constexpr bool regTypeIsGPR = std::is_same<RegType, GPRReg>::value;
if (source.type == Source::Type::BufferOffset) {
regToStore = temp1<RegType>();
load(srcOffset - m_srcOffsetAdjustment, regToStore);
} else if (source.type == Source::Type::Reg)
regToStore = source.getReg<RegType>();
else if constexpr (regTypeIsGPR) {
regToStore = temp1<RegType>();
move(source.value, regToStore);
} else
RELEASE_ASSERT_NOT_REACHED();
store(regToStore, m_deferredStoreOffset - m_dstOffsetAdjustment);
m_currentSource = 0;
}
public:
ALWAYS_INLINE void loadGPR(int srcOffset) { load<GPRReg>(srcOffset); }
ALWAYS_INLINE void copyGPR(GPRReg gpr) { copy<GPRReg>(gpr); }
ALWAYS_INLINE void moveConstant(EncodedJSValue value) { move(value); }
ALWAYS_INLINE void storeGPR(int dstOffset) { store<GPRReg>(dstOffset); }
ALWAYS_INLINE void finalizeGPR() { finalize<GPRReg>(); }
ALWAYS_INLINE void loadFPR(int srcOffset) { load<FPRReg>(srcOffset); }
ALWAYS_INLINE void copyFPR(FPRReg gpr) { copy<FPRReg>(gpr); }
ALWAYS_INLINE void storeFPR(int dstOffset) { store<FPRReg>(dstOffset); }
ALWAYS_INLINE void finalizeFPR() { finalize<FPRReg>(); }
protected:
#if USE(JSVALUE64)
ALWAYS_INLINE void move(EncodedJSValue value, GPRReg dest)
{
m_jit.move(TrustedImm64(value), dest);
}
#else
NO_RETURN_DUE_TO_CRASH void move(EncodedJSValue, GPRReg) { RELEASE_ASSERT_NOT_REACHED(); }
#endif
ALWAYS_INLINE void load(int offset, GPRReg dest)
{
m_jit.loadPtr(Address(m_srcBufferGPR, offset), dest);
}
ALWAYS_INLINE void store(GPRReg src, int offset)
{
m_jit.storePtr(src, Address(m_dstBufferGPR, offset));
}
ALWAYS_INLINE void load(int offset, FPRReg dest)
{
m_jit.loadDouble(Address(m_srcBufferGPR, offset), dest);
}
ALWAYS_INLINE void store(FPRReg src, int offset)
{
m_jit.storeDouble(src, Address(m_dstBufferGPR, offset));
}
#if CPU(ARM64)
template<typename RegType>
ALWAYS_INLINE void loadPair(int offset, RegType dest1, RegType dest2)
{
m_jit.loadPair64(m_srcBufferGPR, TrustedImm32(offset), dest1, dest2);
}
template<typename RegType>
ALWAYS_INLINE void storePair(RegType src1, RegType src2, int offset)
{
m_jit.storePair64(src1, src2, m_dstBufferGPR, TrustedImm32(offset));
}
static constexpr bool hasPairOp = true;
#else
template<typename RegType> ALWAYS_INLINE void loadPair(int, RegType, RegType) { }
template<typename RegType> ALWAYS_INLINE void storePair(RegType, RegType, int) { }
static constexpr bool hasPairOp = false;
#endif
JIT& m_jit;
GPRReg m_srcBufferGPR;
GPRReg m_dstBufferGPR;
GPRReg m_temp1GPR;
GPRReg m_temp2GPR;
FPRReg m_temp1FPR;
FPRReg m_temp2FPR;
private:
static constexpr int gprSize = static_cast<int>(sizeof(CPURegister));
static constexpr int fprSize = static_cast<int>(sizeof(double));
// These point to which register to use.
GPRReg m_gprToStore { InvalidGPRReg }; // Only used when !hasPairOp.
FPRReg m_fprToStore { InvalidFPRReg }; // Only used when !hasPairOp.
BufferRegs m_bufferRegsAttr;
Source m_sources[2];
unsigned m_currentSource { 0 };
int m_srcOffsetAdjustment { 0 };
int m_dstOffsetAdjustment { 0 };
int m_deferredStoreOffset;
template<typename RegType> friend struct RegDispatch;
};
} // namespace JSC
#endif // ENABLE(JIT)