blob: 8084d771748caf4325a735e57c42de2e997bdc6f [file] [log] [blame]
/*
* Copyright (C) 2021 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include "ExceptionExpectation.h"
#include "HashMapImpl.h"
#include "JSCJSValueInlines.h"
#include "VMTrapsInlines.h"
namespace JSC {
ALWAYS_INLINE bool areKeysEqual(JSGlobalObject* globalObject, JSValue a, JSValue b)
{
// We want +0 and -0 to be compared to true here. sameValue() itself doesn't
// guarantee that, however, we normalize all keys before comparing and storing
// them in the map. The normalization will convert -0.0 and 0.0 to the integer
// representation for 0.
return sameValue(globalObject, a, b);
}
// Note that normalization is inlined in DFG's NormalizeMapKey.
// Keep in sync with the implementation of DFG and FTL normalization.
ALWAYS_INLINE JSValue normalizeMapKey(JSValue key)
{
if (!key.isNumber()) {
if (key.isHeapBigInt())
return tryConvertToBigInt32(key.asHeapBigInt());
return key;
}
if (key.isInt32())
return key;
double d = key.asDouble();
if (std::isnan(d))
return jsNaN();
int i = static_cast<int>(d);
if (i == d) {
// When a key is -0, we convert it to positive zero.
// When a key is the double representation for an integer, we convert it to an integer.
return jsNumber(i);
}
// This means key is definitely not negative zero, and it's definitely not a double representation of an integer.
return key;
}
ALWAYS_INLINE uint32_t wangsInt64Hash(uint64_t key)
{
key += ~(key << 32);
key ^= (key >> 22);
key += ~(key << 13);
key ^= (key >> 8);
key += (key << 3);
key ^= (key >> 15);
key += ~(key << 27);
key ^= (key >> 31);
return static_cast<unsigned>(key);
}
ALWAYS_INLINE uint32_t jsMapHash(JSBigInt* bigInt)
{
return bigInt->hash();
}
template<ExceptionExpectation expection>
ALWAYS_INLINE uint32_t jsMapHashImpl(JSGlobalObject* globalObject, VM& vm, JSValue value)
{
ASSERT_WITH_MESSAGE(normalizeMapKey(value) == value, "We expect normalized values flowing into this function.");
if (value.isString()) {
auto scope = DECLARE_THROW_SCOPE(vm);
String wtfString = asString(value)->value(globalObject);
if constexpr (expection == ExceptionExpectation::CanThrow)
RETURN_IF_EXCEPTION(scope, UINT_MAX);
else
EXCEPTION_ASSERT_UNUSED(scope, !scope.exception());
return wtfString.impl()->hash();
}
if (value.isHeapBigInt())
return jsMapHash(value.asHeapBigInt());
return wangsInt64Hash(JSValue::encode(value));
}
ALWAYS_INLINE uint32_t jsMapHash(JSGlobalObject* globalObject, VM& vm, JSValue value)
{
return jsMapHashImpl<ExceptionExpectation::CanThrow>(globalObject, vm, value);
}
ALWAYS_INLINE uint32_t jsMapHashForAlreadyHashedValue(JSGlobalObject* globalObject, VM& vm, JSValue value)
{
return jsMapHashImpl<ExceptionExpectation::ShouldNotThrow>(globalObject, vm, value);
}
ALWAYS_INLINE std::optional<uint32_t> concurrentJSMapHash(JSValue key)
{
key = normalizeMapKey(key);
if (key.isString()) {
JSString* string = asString(key);
if (string->length() > 10 * 1024)
return std::nullopt;
const StringImpl* impl = string->tryGetValueImpl();
if (!impl)
return std::nullopt;
return impl->concurrentHash();
}
if (key.isHeapBigInt())
return key.asHeapBigInt()->concurrentHash();
uint64_t rawValue = JSValue::encode(key);
return wangsInt64Hash(rawValue);
}
ALWAYS_INLINE uint32_t shouldShrink(uint32_t capacity, uint32_t keyCount)
{
return 8 * keyCount <= capacity && capacity > 4;
}
ALWAYS_INLINE uint32_t shouldRehashAfterAdd(uint32_t capacity, uint32_t keyCount, uint32_t deleteCount)
{
return 2 * (keyCount + deleteCount) >= capacity;
}
ALWAYS_INLINE uint32_t nextCapacity(uint32_t capacity, uint32_t keyCount)
{
if (shouldShrink(capacity, keyCount)) {
ASSERT((capacity / 2) >= 4);
return capacity / 2;
}
if (3 * keyCount <= capacity && capacity > 64) {
// We stay at the same size if rehashing would cause us to be no more than
// 1/3rd full. This comes up for programs like this:
// Say the hash table grew to a key count of 64, causing it to grow to a capacity of 256.
// Then, the table added 63 items. The load is now 127. Then, 63 items are deleted.
// The load is still 127. Then, another item is added. The load is now 128, and we
// decide that we need to rehash. The key count is 65, almost exactly what it was
// when we grew to a capacity of 256. We don't really need to grow to a capacity
// of 512 in this situation. Instead, we choose to rehash at the same size. This
// will bring the load down to 65. We rehash into the same size when we determine
// that the new load ratio will be under 1/3rd. (We also pick a minumum capacity
// at which this rule kicks in because otherwise we will be too sensitive to rehashing
// at the same capacity).
return capacity;
}
return Checked<uint32_t>(capacity) * 2;
}
template <typename HashMapBucketType>
void HashMapImpl<HashMapBucketType>::finishCreation(JSGlobalObject* globalObject, VM& vm)
{
ASSERT_WITH_MESSAGE(HashMapBucket<HashMapBucketDataKey>::offsetOfKey() == HashMapBucket<HashMapBucketDataKeyValue>::offsetOfKey(), "We assume this to be true in the DFG and FTL JIT.");
auto scope = DECLARE_THROW_SCOPE(vm);
Base::finishCreation(vm);
makeAndSetNewBuffer(globalObject, vm);
RETURN_IF_EXCEPTION(scope, void());
setUpHeadAndTail(globalObject, vm);
}
template <typename HashMapBucketType>
void HashMapImpl<HashMapBucketType>::finishCreation(JSGlobalObject* globalObject, VM& vm, HashMapImpl* base)
{
auto scope = DECLARE_THROW_SCOPE(vm);
Base::finishCreation(vm);
// This size should be the same to the case when you clone the map by calling add() repeatedly.
uint32_t capacity = (Checked<uint32_t>(base->m_keyCount) * 2) + 1;
RELEASE_ASSERT(capacity <= (1U << 31));
capacity = std::max<uint32_t>(WTF::roundUpToPowerOfTwo(capacity), 4U);
m_capacity = capacity;
makeAndSetNewBuffer(globalObject, vm);
RETURN_IF_EXCEPTION(scope, void());
setUpHeadAndTail(globalObject, vm);
HashMapBucketType* bucket = base->m_head.get()->next();
while (bucket) {
if (!bucket->deleted()) {
addNormalizedNonExistingForCloning(globalObject, bucket->key(), HashMapBucketType::extractValue(*bucket));
RETURN_IF_EXCEPTION(scope, void());
}
bucket = bucket->next();
}
checkConsistency();
}
template <typename HashMapBucketType>
ALWAYS_INLINE HashMapBucketType** HashMapImpl<HashMapBucketType>::findBucket(JSGlobalObject* globalObject, JSValue key)
{
VM& vm = getVM(globalObject);
auto scope = DECLARE_THROW_SCOPE(vm);
key = normalizeMapKey(key);
uint32_t hash = jsMapHash(globalObject, vm, key);
RETURN_IF_EXCEPTION(scope, nullptr);
return findBucket(globalObject, key, hash);
}
template <typename HashMapBucketType>
ALWAYS_INLINE HashMapBucketType** HashMapImpl<HashMapBucketType>::findBucket(JSGlobalObject* globalObject, JSValue key, uint32_t hash)
{
ASSERT_WITH_MESSAGE(normalizeMapKey(key) == key, "We expect normalized values flowing into this function.");
return findBucketAlreadyHashedAndNormalized(globalObject, key, hash);
}
template <typename HashMapBucketType>
template <typename T>
ALWAYS_INLINE typename std::enable_if<std::is_same<T, HashMapBucket<HashMapBucketDataKeyValue>>::value, JSValue>::type HashMapImpl<HashMapBucketType>::get(JSGlobalObject* globalObject, JSValue key)
{
if (HashMapBucketType** bucket = findBucket(globalObject, key))
return (*bucket)->value();
return jsUndefined();
}
template <typename HashMapBucketType>
ALWAYS_INLINE bool HashMapImpl<HashMapBucketType>::has(JSGlobalObject* globalObject, JSValue key)
{
return !!findBucket(globalObject, key);
}
template <typename HashMapBucketType>
ALWAYS_INLINE void HashMapImpl<HashMapBucketType>::add(JSGlobalObject* globalObject, JSValue key, JSValue value)
{
VM& vm = getVM(globalObject);
auto scope = DECLARE_THROW_SCOPE(vm);
key = normalizeMapKey(key);
addNormalizedInternal(globalObject, key, value, [&] (HashMapBucketType* bucket) {
return !isDeleted(bucket) && areKeysEqual(globalObject, key, bucket->key());
});
RETURN_IF_EXCEPTION(scope, void());
scope.release();
if (shouldRehashAfterAdd())
rehash(globalObject);
}
template <typename HashMapBucketType>
ALWAYS_INLINE HashMapBucketType* HashMapImpl<HashMapBucketType>::addNormalized(JSGlobalObject* globalObject, JSValue key, JSValue value, uint32_t hash)
{
VM& vm = getVM(globalObject);
ASSERT_WITH_MESSAGE(normalizeMapKey(key) == key, "We expect normalized values flowing into this function.");
DEFER_TERMINATION_AND_ASSERT_WITH_MESSAGE(vm, jsMapHash(globalObject, getVM(globalObject), key) == hash, "We expect hash value is what we expect.");
auto* bucket = addNormalizedInternal(vm, key, value, hash, [&] (HashMapBucketType* bucket) {
return !isDeleted(bucket) && areKeysEqual(globalObject, key, bucket->key());
});
if (shouldRehashAfterAdd())
rehash(globalObject);
return bucket;
}
template <typename HashMapBucketType>
ALWAYS_INLINE bool HashMapImpl<HashMapBucketType>::remove(JSGlobalObject* globalObject, JSValue key)
{
HashMapBucketType** bucket = findBucket(globalObject, key);
if (!bucket)
return false;
VM& vm = getVM(globalObject);
HashMapBucketType* impl = *bucket;
impl->next()->setPrev(vm, impl->prev());
impl->prev()->setNext(vm, impl->next());
impl->makeDeleted(vm);
*bucket = deletedValue();
++m_deleteCount;
ASSERT(m_keyCount > 0);
--m_keyCount;
if (shouldShrink())
rehash(globalObject);
return true;
}
template <typename HashMapBucketType>
ALWAYS_INLINE void HashMapImpl<HashMapBucketType>::clear(JSGlobalObject* globalObject)
{
VM& vm = getVM(globalObject);
m_keyCount = 0;
m_deleteCount = 0;
HashMapBucketType* head = m_head.get();
HashMapBucketType* bucket = m_head->next();
HashMapBucketType* tail = m_tail.get();
while (bucket != tail) {
HashMapBucketType* next = bucket->next();
// We restart each iterator by pointing it to the head of the list.
bucket->setNext(vm, head);
bucket->makeDeleted(vm);
bucket = next;
}
m_head->setNext(vm, m_tail.get());
m_tail->setPrev(vm, m_head.get());
m_capacity = 4;
makeAndSetNewBuffer(globalObject, vm);
checkConsistency();
}
template <typename HashMapBucketType>
ALWAYS_INLINE void HashMapImpl<HashMapBucketType>::setUpHeadAndTail(JSGlobalObject*, VM& vm)
{
m_head.set(vm, this, HashMapBucketType::create(vm));
m_tail.set(vm, this, HashMapBucketType::create(vm));
m_head->setNext(vm, m_tail.get());
m_tail->setPrev(vm, m_head.get());
ASSERT(m_head->deleted());
ASSERT(m_tail->deleted());
}
template <typename HashMapBucketType>
ALWAYS_INLINE void HashMapImpl<HashMapBucketType>::addNormalizedNonExistingForCloning(JSGlobalObject* globalObject, JSValue key, JSValue value)
{
addNormalizedInternal(globalObject, key, value, [&] (HashMapBucketType*) {
return false;
});
}
template <typename HashMapBucketType>
template<typename CanUseBucket>
ALWAYS_INLINE void HashMapImpl<HashMapBucketType>::addNormalizedInternal(JSGlobalObject* globalObject, JSValue key, JSValue value, const CanUseBucket& canUseBucket)
{
VM& vm = getVM(globalObject);
auto scope = DECLARE_THROW_SCOPE(vm);
uint32_t hash = jsMapHash(globalObject, vm, key);
RETURN_IF_EXCEPTION(scope, void());
scope.release();
addNormalizedInternal(vm, key, value, hash, canUseBucket);
}
template <typename HashMapBucketType>
template<typename CanUseBucket>
ALWAYS_INLINE HashMapBucketType* HashMapImpl<HashMapBucketType>::addNormalizedInternal(VM& vm, JSValue key, JSValue value, uint32_t hash, const CanUseBucket& canUseBucket)
{
ASSERT_WITH_MESSAGE(normalizeMapKey(key) == key, "We expect normalized values flowing into this function.");
const uint32_t mask = m_capacity - 1;
uint32_t index = hash & mask;
HashMapBucketType** buffer = this->buffer();
HashMapBucketType* bucket = buffer[index];
while (!isEmpty(bucket)) {
if (canUseBucket(bucket)) {
bucket->setValue(vm, value);
return bucket;
}
index = (index + 1) & mask;
bucket = buffer[index];
}
HashMapBucketType* newEntry = m_tail.get();
buffer[index] = newEntry;
newEntry->setKey(vm, key);
newEntry->setValue(vm, value);
ASSERT(!newEntry->deleted());
HashMapBucketType* newTail = HashMapBucketType::create(vm);
m_tail.set(vm, this, newTail);
newTail->setPrev(vm, newEntry);
ASSERT(newTail->deleted());
newEntry->setNext(vm, newTail);
++m_keyCount;
return newEntry;
}
template <typename HashMapBucketType>
ALWAYS_INLINE HashMapBucketType** HashMapImpl<HashMapBucketType>::findBucketAlreadyHashedAndNormalized(JSGlobalObject* globalObject, JSValue key, uint32_t hash)
{
const uint32_t mask = m_capacity - 1;
uint32_t index = hash & mask;
HashMapBucketType** buffer = this->buffer();
HashMapBucketType* bucket = buffer[index];
while (!isEmpty(bucket)) {
if (!isDeleted(bucket) && areKeysEqual(globalObject, key, bucket->key()))
return buffer + index;
index = (index + 1) & mask;
bucket = buffer[index];
}
return nullptr;
}
template <typename HashMapBucketType>
void HashMapImpl<HashMapBucketType>::rehash(JSGlobalObject* globalObject)
{
VM& vm = getVM(globalObject);
auto scope = DECLARE_THROW_SCOPE(vm);
uint32_t oldCapacity = m_capacity;
m_capacity = nextCapacity(m_capacity, m_keyCount);
if (m_capacity != oldCapacity) {
makeAndSetNewBuffer(globalObject, vm);
RETURN_IF_EXCEPTION(scope, void());
} else {
m_buffer->reset(m_capacity);
assertBufferIsEmpty();
}
HashMapBucketType* iter = m_head->next();
HashMapBucketType* end = m_tail.get();
const uint32_t mask = m_capacity - 1;
RELEASE_ASSERT(!(m_capacity & (m_capacity - 1)));
HashMapBucketType** buffer = this->buffer();
while (iter != end) {
uint32_t index = jsMapHashForAlreadyHashedValue(globalObject, vm, iter->key()) & mask;
EXCEPTION_ASSERT_WITH_MESSAGE(!scope.exception(), "All keys should already be hashed before, so this should not throw because it won't resolve ropes.");
{
HashMapBucketType* bucket = buffer[index];
while (!isEmpty(bucket)) {
index = (index + 1) & mask;
bucket = buffer[index];
}
}
buffer[index] = iter;
iter = iter->next();
}
m_deleteCount = 0;
checkConsistency();
}
template <typename HashMapBucketType>
ALWAYS_INLINE void HashMapImpl<HashMapBucketType>::checkConsistency() const
{
if (ASSERT_ENABLED) {
HashMapBucketType* iter = m_head->next();
HashMapBucketType* end = m_tail.get();
uint32_t size = 0;
while (iter != end) {
++size;
iter = iter->next();
}
ASSERT(size == m_keyCount);
}
}
template <typename HashMapBucketType>
void HashMapImpl<HashMapBucketType>::makeAndSetNewBuffer(JSGlobalObject* globalObject, VM& vm)
{
ASSERT(!(m_capacity & (m_capacity - 1)));
HashMapBufferType* buffer = HashMapBufferType::create(globalObject, vm, this, m_capacity);
if (UNLIKELY(!buffer))
return;
m_buffer.set(vm, this, buffer);
assertBufferIsEmpty();
}
template <typename HashMapBucketType>
ALWAYS_INLINE void HashMapImpl<HashMapBucketType>::assertBufferIsEmpty() const
{
if (ASSERT_ENABLED) {
for (unsigned i = 0; i < m_capacity; i++)
ASSERT(isEmpty(buffer()[i]));
}
}
} // namespace JSC