Diff
Modified: trunk/Source/bmalloc/ChangeLog (242937 => 242938)
--- trunk/Source/bmalloc/ChangeLog 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/ChangeLog 2019-03-14 08:02:00 UTC (rev 242938)
@@ -1,3 +1,90 @@
+2019-03-14 Yusuke Suzuki <[email protected]>
+
+ [bmalloc] Add StaticPerProcess for known types to save pages
+ https://bugs.webkit.org/show_bug.cgi?id=195691
+
+ Reviewed by Mark Lam.
+
+ As initial memory footprint of VM + JSGlobalObject becomes 488KB dirty size in fast malloc memory (w/ JSC_useJIT=0 and Malloc=1), pages for PerProcess is costly.
+ For example, under Malloc=1 mode, we still need to allocate PerProcess<DebugHeap> and PerProcess<Environment>. And sizeof(Environment) is only 1 (bool flag), and
+ sizeof(DebugHeap) is 120. But we are allocating 1 pages for them. Since page size in iOS is 16KB, this 121B consumes 16KB dirty memory, and it is not negligible
+ size if we keep in mind that the current fast malloc heap size is 488KB. Putting them into the __DATA section, close to the other mutable data, we can avoid allocating
+ this page.
+
+ This patch revives the SafePerProcess concept in r228107. We add "StaticPerProcess<T>", which allocates underlying storage statically in the __DATA section instead of
+ allocating it at runtime. And we use this StaticPerProcess<T> for types where (1) T is known a priori, and (2) sizeof(T) is not huge.
+
+ * bmalloc.xcodeproj/project.pbxproj:
+ * bmalloc/AllIsoHeaps.cpp:
+ * bmalloc/AllIsoHeaps.h:
+ * bmalloc/Allocator.cpp:
+ (bmalloc::Allocator::Allocator):
+ * bmalloc/Cache.cpp:
+ (bmalloc::Cache::Cache):
+ * bmalloc/CryptoRandom.cpp:
+ (bmalloc::cryptoRandom):
+ * bmalloc/Deallocator.cpp:
+ (bmalloc::Deallocator::Deallocator):
+ * bmalloc/DebugHeap.cpp:
+ * bmalloc/DebugHeap.h:
+ (bmalloc::DebugHeap::tryGet):
+ * bmalloc/Environment.cpp:
+ * bmalloc/Environment.h:
+ * bmalloc/Gigacage.cpp:
+ (Gigacage::Callback::Callback):
+ (Gigacage::Callback::function):
+ (bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks):
+ (Gigacage::disablePrimitiveGigacage):
+ (Gigacage::addPrimitiveDisableCallback):
+ (Gigacage::removePrimitiveDisableCallback):
+ (Gigacage::shouldBeEnabled):
+ (Gigacage::bmalloc::Callback::Callback): Deleted.
+ (Gigacage::bmalloc::Callback::function): Deleted.
+ (Gigacage::bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks): Deleted.
+ * bmalloc/Heap.cpp:
+ (bmalloc::Heap::Heap):
+ (bmalloc::Heap::tryAllocateLarge):
+ * bmalloc/IsoDirectoryInlines.h:
+ (bmalloc::passedNumPages>::takeFirstEligible):
+ (bmalloc::passedNumPages>::didBecome):
+ * bmalloc/IsoHeapImpl.cpp:
+ (bmalloc::IsoHeapImplBase::addToAllIsoHeaps):
+ * bmalloc/IsoPage.cpp:
+ (bmalloc::IsoPageBase::allocatePageMemory):
+ * bmalloc/IsoTLS.cpp:
+ (bmalloc::IsoTLS::IsoTLS):
+ (bmalloc::IsoTLS::ensureEntries):
+ (bmalloc::IsoTLS::forEachEntry):
+ * bmalloc/IsoTLSEntry.cpp:
+ (bmalloc::IsoTLSEntry::IsoTLSEntry):
+ * bmalloc/IsoTLSInlines.h:
+ (bmalloc::IsoTLS::allocateSlow):
+ (bmalloc::IsoTLS::deallocateSlow):
+ * bmalloc/IsoTLSLayout.cpp:
+ * bmalloc/IsoTLSLayout.h:
+ * bmalloc/Scavenger.cpp:
+ (bmalloc::Scavenger::Scavenger):
+ (bmalloc::dumpStats):
+ (bmalloc::Scavenger::scavenge):
+ (bmalloc::Scavenger::partialScavenge):
+ (bmalloc::Scavenger::freeableMemory):
+ (bmalloc::Scavenger::footprint):
+ * bmalloc/Scavenger.h:
+ * bmalloc/StaticPerProcess.h: Added.
+ * bmalloc/VMHeap.cpp:
+ * bmalloc/VMHeap.h:
+ * bmalloc/Zone.h:
+ * bmalloc/bmalloc.cpp:
+ (bmalloc::api::scavenge):
+ (bmalloc::api::isEnabled):
+ (bmalloc::api::setScavengerThreadQOSClass):
+ (bmalloc::api::enableMiniMode):
+ * test/testbmalloc.cpp:
+ (assertEmptyPointerSet):
+ (assertHasObjects):
+ (assertHasOnlyObjects):
+ (assertClean):
+
2019-03-13 Yoshiaki Jitsukawa <[email protected]>
[bmalloc] Use MADV_FREE on FreeBSD
Modified: trunk/Source/bmalloc/bmalloc/AllIsoHeaps.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/AllIsoHeaps.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/AllIsoHeaps.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -27,6 +27,8 @@
namespace bmalloc {
+DEFINE_STATIC_PER_PROCESS_STORAGE(AllIsoHeaps);
+
AllIsoHeaps::AllIsoHeaps(const std::lock_guard<Mutex>&)
{
}
Modified: trunk/Source/bmalloc/bmalloc/AllIsoHeaps.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/AllIsoHeaps.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/AllIsoHeaps.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -26,11 +26,12 @@
#pragma once
#include "IsoHeapImpl.h"
+#include "StaticPerProcess.h"
#include "Vector.h"
namespace bmalloc {
-class AllIsoHeaps {
+class AllIsoHeaps : public StaticPerProcess<AllIsoHeaps> {
public:
AllIsoHeaps(const std::lock_guard<Mutex>&);
@@ -44,6 +45,7 @@
Mutex m_lock;
IsoHeapImplBase* m_head { nullptr };
};
+DECLARE_STATIC_PER_PROCESS_STORAGE(AllIsoHeaps);
} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/Allocator.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Allocator.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Allocator.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -40,7 +40,7 @@
: m_heap(heap)
, m_deallocator(deallocator)
{
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
for (size_t sizeClass = 0; sizeClass < sizeClassCount; ++sizeClass)
m_bumpAllocators[sizeClass].init(objectSize(sizeClass));
}
Modified: trunk/Source/bmalloc/bmalloc/Cache.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Cache.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Cache.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -48,7 +48,7 @@
: m_deallocator(PerProcess<PerHeapKind<Heap>>::get()->at(heapKind))
, m_allocator(PerProcess<PerHeapKind<Heap>>::get()->at(heapKind), m_deallocator)
{
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
}
BNO_INLINE void* Cache::tryAllocateSlowCaseNullCache(HeapKind heapKind, size_t size)
Modified: trunk/Source/bmalloc/bmalloc/CryptoRandom.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/CryptoRandom.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/CryptoRandom.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -33,7 +33,7 @@
#include "BAssert.h"
#include "BPlatform.h"
#include "Mutex.h"
-#include "PerProcess.h"
+#include "StaticPerProcess.h"
#include "VMAllocate.h"
#include <mutex>
@@ -59,7 +59,7 @@
uint8_t s[256];
};
-class ARC4RandomNumberGenerator {
+class ARC4RandomNumberGenerator : public StaticPerProcess<ARC4RandomNumberGenerator> {
public:
ARC4RandomNumberGenerator(const std::lock_guard<Mutex>&);
@@ -76,6 +76,8 @@
int m_count;
Mutex m_mutex;
};
+DECLARE_STATIC_PER_PROCESS_STORAGE(ARC4RandomNumberGenerator);
+DEFINE_STATIC_PER_PROCESS_STORAGE(ARC4RandomNumberGenerator);
ARC4Stream::ARC4Stream()
{
@@ -176,7 +178,7 @@
void cryptoRandom(void* buffer, size_t length)
{
- PerProcess<ARC4RandomNumberGenerator>::get()->randomValues(buffer, length);
+ ARC4RandomNumberGenerator::get()->randomValues(buffer, length);
}
} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/Deallocator.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Deallocator.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Deallocator.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -40,7 +40,7 @@
Deallocator::Deallocator(Heap& heap)
: m_heap(heap)
{
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
}
Deallocator::~Deallocator()
Modified: trunk/Source/bmalloc/bmalloc/DebugHeap.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/DebugHeap.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/DebugHeap.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -36,6 +36,8 @@
DebugHeap* debugHeapCache { nullptr };
+DEFINE_STATIC_PER_PROCESS_STORAGE(DebugHeap);
+
#if BOS(DARWIN)
DebugHeap::DebugHeap(std::lock_guard<Mutex>&)
Modified: trunk/Source/bmalloc/bmalloc/DebugHeap.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/DebugHeap.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/DebugHeap.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -27,7 +27,7 @@
#include "Environment.h"
#include "Mutex.h"
-#include "PerProcess.h"
+#include "StaticPerProcess.h"
#include <mutex>
#include <unordered_map>
@@ -37,7 +37,7 @@
namespace bmalloc {
-class DebugHeap {
+class DebugHeap : private StaticPerProcess<DebugHeap> {
public:
DebugHeap(std::lock_guard<Mutex>&);
@@ -64,6 +64,7 @@
std::mutex m_lock;
std::unordered_map<void*, size_t> m_sizeMap;
};
+DECLARE_STATIC_PER_PROCESS_STORAGE(DebugHeap);
extern BEXPORT DebugHeap* debugHeapCache;
BINLINE DebugHeap* DebugHeap::tryGet()
@@ -70,8 +71,8 @@
{
if (debugHeapCache)
return debugHeapCache;
- if (PerProcess<Environment>::get()->isDebugHeapEnabled()) {
- debugHeapCache = PerProcess<DebugHeap>::get();
+ if (Environment::get()->isDebugHeapEnabled()) {
+ debugHeapCache = DebugHeap::get();
return debugHeapCache;
}
return nullptr;
Modified: trunk/Source/bmalloc/bmalloc/Environment.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Environment.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Environment.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -125,6 +125,8 @@
}
#endif
+DEFINE_STATIC_PER_PROCESS_STORAGE(Environment);
+
Environment::Environment(std::lock_guard<Mutex>&)
: m_isDebugHeapEnabled(computeIsDebugHeapEnabled())
{
Modified: trunk/Source/bmalloc/bmalloc/Environment.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Environment.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Environment.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -27,10 +27,11 @@
#define Environment_h
#include "Mutex.h"
+#include "StaticPerProcess.h"
namespace bmalloc {
-class Environment {
+class Environment : public StaticPerProcess<Environment> {
public:
BEXPORT Environment(std::lock_guard<Mutex>&);
@@ -41,6 +42,7 @@
bool m_isDebugHeapEnabled;
};
+DECLARE_STATIC_PER_PROCESS_STORAGE(Environment);
} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/Gigacage.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Gigacage.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Gigacage.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -27,8 +27,8 @@
#include "CryptoRandom.h"
#include "Environment.h"
-#include "PerProcess.h"
#include "ProcessCheck.h"
+#include "StaticPerProcess.h"
#include "VMAllocate.h"
#include "Vector.h"
#include "bmalloc.h"
@@ -39,6 +39,35 @@
namespace Gigacage {
+struct Callback {
+ Callback() { }
+
+ Callback(void (*function)(void*), void *argument)
+ : function(function)
+ , argument(argument)
+ {
+ }
+
+ void (*function)(void*) { nullptr };
+ void* argument { nullptr };
+};
+
+}
+
+namespace bmalloc {
+
+struct PrimitiveDisableCallbacks : public StaticPerProcess<PrimitiveDisableCallbacks> {
+ PrimitiveDisableCallbacks(std::lock_guard<Mutex>&) { }
+
+ Vector<Gigacage::Callback> callbacks;
+};
+DECLARE_STATIC_PER_PROCESS_STORAGE(PrimitiveDisableCallbacks);
+DEFINE_STATIC_PER_PROCESS_STORAGE(PrimitiveDisableCallbacks);
+
+} // namespace bmalloc
+
+namespace Gigacage {
+
// This is exactly 32GB because inside JSC, indexed accesses for arrays, typed arrays, etc,
// use unsigned 32-bit ints as indices. The items those indices access are 8 bytes or less
// in size. 2^32 * 8 = 32GB. This means if an access on a caged type happens to go out of
@@ -84,25 +113,6 @@
}
};
-struct Callback {
- Callback() { }
-
- Callback(void (*function)(void*), void *argument)
- : function(function)
- , argument(argument)
- {
- }
-
- void (*function)(void*) { nullptr };
- void* argument { nullptr };
-};
-
-struct PrimitiveDisableCallbacks {
- PrimitiveDisableCallbacks(std::lock_guard<Mutex>&) { }
-
- Vector<Callback> callbacks;
-};
-
size_t runwaySize(Kind kind)
{
switch (kind) {
@@ -199,8 +209,8 @@
return;
}
- PrimitiveDisableCallbacks& callbacks = *PerProcess<PrimitiveDisableCallbacks>::get();
- std::unique_lock<Mutex> lock(PerProcess<PrimitiveDisableCallbacks>::mutex());
+ PrimitiveDisableCallbacks& callbacks = *PrimitiveDisableCallbacks::get();
+ std::unique_lock<Mutex> lock(PrimitiveDisableCallbacks::mutex());
for (Callback& callback : callbacks.callbacks)
callback.function(callback.argument);
callbacks.callbacks.shrink(0);
@@ -217,15 +227,15 @@
return;
}
- PrimitiveDisableCallbacks& callbacks = *PerProcess<PrimitiveDisableCallbacks>::get();
- std::unique_lock<Mutex> lock(PerProcess<PrimitiveDisableCallbacks>::mutex());
+ PrimitiveDisableCallbacks& callbacks = *PrimitiveDisableCallbacks::get();
+ std::unique_lock<Mutex> lock(PrimitiveDisableCallbacks::mutex());
callbacks.callbacks.push(Callback(function, argument));
}
void removePrimitiveDisableCallback(void (*function)(void*), void* argument)
{
- PrimitiveDisableCallbacks& callbacks = *PerProcess<PrimitiveDisableCallbacks>::get();
- std::unique_lock<Mutex> lock(PerProcess<PrimitiveDisableCallbacks>::mutex());
+ PrimitiveDisableCallbacks& callbacks = *PrimitiveDisableCallbacks::get();
+ std::unique_lock<Mutex> lock(PrimitiveDisableCallbacks::mutex());
for (size_t i = 0; i < callbacks.callbacks.size(); ++i) {
if (callbacks.callbacks[i].function == function
&& callbacks.callbacks[i].argument == argument) {
@@ -267,7 +277,7 @@
std::call_once(
onceFlag,
[] {
- bool debugHeapEnabled = PerProcess<Environment>::get()->isDebugHeapEnabled();
+ bool debugHeapEnabled = Environment::get()->isDebugHeapEnabled();
if (debugHeapEnabled)
return;
Modified: trunk/Source/bmalloc/bmalloc/Heap.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Heap.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Heap.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -54,7 +54,7 @@
initializeLineMetadata();
initializePageMetadata();
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
Gigacage::ensureGigacage();
#if GIGACAGE_ENABLED
@@ -69,7 +69,7 @@
}
#endif
- m_scavenger = PerProcess<Scavenger>::get();
+ m_scavenger = Scavenger::get();
}
bool Heap::usingGigacage()
@@ -574,7 +574,7 @@
if (usingGigacage())
return nullptr;
- range = PerProcess<VMHeap>::get()->tryAllocateLargeChunk(alignment, size);
+ range = VMHeap::get()->tryAllocateLargeChunk(alignment, size);
if (!range)
return nullptr;
Modified: trunk/Source/bmalloc/bmalloc/IsoDirectoryInlines.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoDirectoryInlines.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoDirectoryInlines.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -53,7 +53,7 @@
m_highWatermark = std::max(pageIndex, m_highWatermark);
- Scavenger& scavenger = *PerProcess<Scavenger>::get();
+ Scavenger& scavenger = *Scavenger::get();
scavenger.didStartGrowing();
IsoPage<Config>* page = m_pages[pageIndex];
@@ -108,7 +108,7 @@
BASSERT(!!m_committed[pageIndex]);
this->m_heap.isNowFreeable(page, IsoPageBase::pageSize);
m_empty[pageIndex] = true;
- PerProcess<Scavenger>::get()->schedule(IsoPageBase::pageSize);
+ Scavenger::get()->schedule(IsoPageBase::pageSize);
return;
}
BCRASH();
Modified: trunk/Source/bmalloc/bmalloc/IsoHeapImpl.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoHeapImpl.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoHeapImpl.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -41,7 +41,7 @@
void IsoHeapImplBase::addToAllIsoHeaps()
{
- PerProcess<AllIsoHeaps>::get()->add(this);
+ AllIsoHeaps::get()->add(this);
}
void IsoHeapImplBase::scavengeNow()
Modified: trunk/Source/bmalloc/bmalloc/IsoPage.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoPage.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoPage.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -32,7 +32,7 @@
void* IsoPageBase::allocatePageMemory()
{
- return PerProcess<VMHeap>::get()->tryAllocateLargeChunk(pageSize, pageSize).begin();
+ return VMHeap::get()->tryAllocateLargeChunk(pageSize, pageSize).begin();
}
} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/IsoTLS.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoTLS.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoTLS.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -54,7 +54,7 @@
IsoTLS::IsoTLS()
{
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
}
IsoTLS* IsoTLS::ensureEntries(unsigned offset)
@@ -77,7 +77,7 @@
});
IsoTLS* tls = get();
- IsoTLSLayout& layout = *PerProcess<IsoTLSLayout>::get();
+ IsoTLSLayout& layout = *IsoTLSLayout::get();
IsoTLSEntry* oldLastEntry = tls ? tls->m_lastEntry : nullptr;
RELEASE_BASSERT(!oldLastEntry || oldLastEntry->offset() < offset);
@@ -167,7 +167,7 @@
{
if (!m_lastEntry)
return;
- PerProcess<IsoTLSLayout>::get()->head()->walkUpToInclusive(
+ IsoTLSLayout::get()->head()->walkUpToInclusive(
m_lastEntry,
[&] (IsoTLSEntry* entry) {
func(entry, m_data + entry->offset());
Modified: trunk/Source/bmalloc/bmalloc/IsoTLSEntry.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoTLSEntry.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoTLSEntry.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -37,7 +37,7 @@
, m_alignment(alignment)
, m_size(size)
{
- PerProcess<IsoTLSLayout>::get()->add(this);
+ IsoTLSLayout::get()->add(this);
RELEASE_BASSERT(m_offset != UINT_MAX);
}
Modified: trunk/Source/bmalloc/bmalloc/IsoTLSInlines.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoTLSInlines.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoTLSInlines.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -96,7 +96,7 @@
}
// If debug heap is enabled, s_mallocFallbackState becomes MallocFallbackState::FallBackToMalloc.
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
IsoTLS* tls = ensureHeapAndEntries(handle);
@@ -139,7 +139,7 @@
}
// If debug heap is enabled, s_mallocFallbackState becomes MallocFallbackState::FallBackToMalloc.
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
RELEASE_BASSERT(handle.isInitialized());
Modified: trunk/Source/bmalloc/bmalloc/IsoTLSLayout.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoTLSLayout.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoTLSLayout.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -29,6 +29,8 @@
namespace bmalloc {
+DEFINE_STATIC_PER_PROCESS_STORAGE(IsoTLSLayout);
+
IsoTLSLayout::IsoTLSLayout(const std::lock_guard<Mutex>&)
{
}
Modified: trunk/Source/bmalloc/bmalloc/IsoTLSLayout.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/IsoTLSLayout.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/IsoTLSLayout.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -26,6 +26,7 @@
#pragma once
#include "Mutex.h"
+#include "StaticPerProcess.h"
#include <mutex>
namespace bmalloc {
@@ -32,7 +33,7 @@
class IsoTLSEntry;
-class IsoTLSLayout {
+class IsoTLSLayout : public StaticPerProcess<IsoTLSLayout> {
public:
IsoTLSLayout(const std::lock_guard<Mutex>&);
@@ -44,6 +45,7 @@
IsoTLSEntry* m_head { nullptr };
IsoTLSEntry* m_tail { nullptr };
};
+DECLARE_STATIC_PER_PROCESS_STORAGE(IsoTLSLayout);
} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/Scavenger.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Scavenger.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Scavenger.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -65,9 +65,11 @@
bool printed { false };
};
+DEFINE_STATIC_PER_PROCESS_STORAGE(Scavenger);
+
Scavenger::Scavenger(std::lock_guard<Mutex>&)
{
- BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ BASSERT(!Environment::get()->isDebugHeapEnabled());
#if BOS(DARWIN)
auto queue = dispatch_queue_create("WebKit Malloc Memory Pressure Handler", DISPATCH_QUEUE_SERIAL);
@@ -165,8 +167,8 @@
}
#endif
- dump("bmalloc-freeable", PerProcess<Scavenger>::get()->freeableMemory());
- dump("bmalloc-footprint", PerProcess<Scavenger>::get()->footprint());
+ dump("bmalloc-freeable", Scavenger::get()->freeableMemory());
+ dump("bmalloc-footprint", Scavenger::get()->footprint());
}
std::chrono::milliseconds Scavenger::timeSinceLastFullScavenge()
@@ -230,7 +232,7 @@
{
RELEASE_BASSERT(!m_deferredDecommits.size());
- PerProcess<AllIsoHeaps>::get()->forEach(
+ AllIsoHeaps::get()->forEach(
[&] (IsoHeapImplBase& heap) {
heap.scavenge(m_deferredDecommits);
});
@@ -297,7 +299,7 @@
{
RELEASE_BASSERT(!m_deferredDecommits.size());
- PerProcess<AllIsoHeaps>::get()->forEach(
+ AllIsoHeaps::get()->forEach(
[&] (IsoHeapImplBase& heap) {
heap.scavengeToHighWatermark(m_deferredDecommits);
});
@@ -329,7 +331,7 @@
}
}
- PerProcess<AllIsoHeaps>::get()->forEach(
+ AllIsoHeaps::get()->forEach(
[&] (IsoHeapImplBase& heap) {
result += heap.freeableMemory();
});
@@ -339,7 +341,7 @@
size_t Scavenger::footprint()
{
- RELEASE_BASSERT(!PerProcess<Environment>::get()->isDebugHeapEnabled());
+ RELEASE_BASSERT(!Environment::get()->isDebugHeapEnabled());
size_t result = 0;
for (unsigned i = numHeaps; i--;) {
@@ -348,7 +350,7 @@
result += PerProcess<PerHeapKind<Heap>>::get()->at(i).footprint();
}
- PerProcess<AllIsoHeaps>::get()->forEach(
+ AllIsoHeaps::get()->forEach(
[&] (IsoHeapImplBase& heap) {
result += heap.footprint();
});
Modified: trunk/Source/bmalloc/bmalloc/Scavenger.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Scavenger.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Scavenger.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -28,7 +28,7 @@
#include "BPlatform.h"
#include "DeferredDecommit.h"
#include "Mutex.h"
-#include "PerProcess.h"
+#include "StaticPerProcess.h"
#include "Vector.h"
#include <chrono>
#include <condition_variable>
@@ -40,7 +40,7 @@
namespace bmalloc {
-class Scavenger {
+class Scavenger : public StaticPerProcess<Scavenger> {
public:
BEXPORT Scavenger(std::lock_guard<Mutex>&);
@@ -112,6 +112,7 @@
Vector<DeferredDecommit> m_deferredDecommits;
};
+DECLARE_STATIC_PER_PROCESS_STORAGE(Scavenger);
} // namespace bmalloc
Added: trunk/Source/bmalloc/bmalloc/StaticPerProcess.h (0 => 242938)
--- trunk/Source/bmalloc/bmalloc/StaticPerProcess.h (rev 0)
+++ trunk/Source/bmalloc/bmalloc/StaticPerProcess.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2014-2019 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "BInline.h"
+#include "Mutex.h"
+#include "Sizes.h"
+
+namespace bmalloc {
+
+// StaticPerProcess<T> behaves like PerProcess<T>, but we need to explicitly define storage for T with EXTERN.
+// In this way, we allocate storage for a per-process object statically instead of allocating memory at runtime.
+// To enforce this, we have DECLARE and DEFINE macros. If you do not know about T of StaticPerProcess<T>, you should use PerProcess<T> instead.
+//
+// Usage:
+// In Object.h
+// class Object : public StaticPerProcess<Object> {
+// ...
+// };
+// DECLARE_STATIC_PER_PROCESS_STORAGE(Object);
+//
+// In Object.cpp
+// DEFINE_STATIC_PER_PROCESS_STORAGE(Object);
+//
+// Object* object = Object::get();
+// x = object->field->field;
+//
+// Object will be instantiated only once, even in the presence of concurrency.
+//
+template<typename T> struct StaticPerProcessStorageTraits;
+
+template<typename T>
+class BEXPORT StaticPerProcess {
+public:
+ static T* get()
+ {
+ T* object = getFastCase();
+ if (!object)
+ return getSlowCase();
+ return object;
+ }
+
+ static T* getFastCase()
+ {
+ using Storage = typename StaticPerProcessStorageTraits<T>::Storage;
+ return (Storage::s_object).load(std::memory_order_relaxed);
+ }
+
+ static Mutex& mutex()
+ {
+ using Storage = typename StaticPerProcessStorageTraits<T>::Storage;
+ return Storage::s_mutex;
+ }
+
+private:
+ BNO_INLINE static T* getSlowCase()
+ {
+ using Storage = typename StaticPerProcessStorageTraits<T>::Storage;
+ std::lock_guard<Mutex> lock(Storage::s_mutex);
+ if (!Storage::s_object.load(std::memory_order_consume)) {
+ T* t = new (&Storage::s_memory) T(lock);
+ Storage::s_object.store(t, std::memory_order_release);
+ }
+ return Storage::s_object.load(std::memory_order_consume);
+ }
+};
+
+#define DECLARE_STATIC_PER_PROCESS_STORAGE(Type) \
+template<> struct StaticPerProcessStorageTraits<Type> { \
+ using Memory = typename std::aligned_storage<sizeof(Type), std::alignment_of<Type>::value>::type; \
+ struct BEXPORT Storage { \
+ BEXPORT static std::atomic<Type*> s_object; \
+ BEXPORT static Mutex s_mutex; \
+ BEXPORT static Memory s_memory; \
+ }; \
+};
+
+#define DEFINE_STATIC_PER_PROCESS_STORAGE(Type) \
+ std::atomic<Type*> StaticPerProcessStorageTraits<Type>::Storage::s_object { nullptr }; \
+ Mutex StaticPerProcessStorageTraits<Type>::Storage::s_mutex { }; \
+ StaticPerProcessStorageTraits<Type>::Memory StaticPerProcessStorageTraits<Type>::Storage::s_memory { };
+
+} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/VMHeap.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/VMHeap.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/VMHeap.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -29,6 +29,8 @@
namespace bmalloc {
+DEFINE_STATIC_PER_PROCESS_STORAGE(VMHeap);
+
VMHeap::VMHeap(std::lock_guard<Mutex>&)
{
}
Modified: trunk/Source/bmalloc/bmalloc/VMHeap.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/VMHeap.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/VMHeap.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -31,6 +31,7 @@
#include "HeapKind.h"
#include "LargeRange.h"
#include "Map.h"
+#include "StaticPerProcess.h"
#include "Vector.h"
#if BOS(DARWIN)
#include "Zone.h"
@@ -44,12 +45,13 @@
typedef enum { Sync, Async } ScavengeMode;
-class VMHeap {
+class VMHeap : public StaticPerProcess<VMHeap> {
public:
VMHeap(std::lock_guard<Mutex>&);
LargeRange tryAllocateLargeChunk(size_t alignment, size_t);
};
+DECLARE_STATIC_PER_PROCESS_STORAGE(VMHeap);
} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/Zone.h (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/Zone.h 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/Zone.h 2019-03-14 08:02:00 UTC (rev 242938)
@@ -29,6 +29,7 @@
#include "FixedVector.h"
#include "Mutex.h"
#include "Range.h"
+#include "StaticPerProcess.h"
#include <malloc/malloc.h>
#include <mutex>
Modified: trunk/Source/bmalloc/bmalloc/bmalloc.cpp (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc/bmalloc.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc/bmalloc.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -95,12 +95,12 @@
if (DebugHeap* debugHeap = DebugHeap::tryGet())
debugHeap->scavenge();
else
- PerProcess<Scavenger>::get()->scavenge();
+ Scavenger::get()->scavenge();
}
bool isEnabled(HeapKind)
{
- return !PerProcess<Environment>::get()->isDebugHeapEnabled();
+ return !Environment::get()->isDebugHeapEnabled();
}
#if BOS(DARWIN)
@@ -109,7 +109,7 @@
if (DebugHeap::tryGet())
return;
std::unique_lock<Mutex> lock(Heap::mutex());
- PerProcess<Scavenger>::get()->setScavengerThreadQOSClass(overrideClass);
+ Scavenger::get()->setScavengerThreadQOSClass(overrideClass);
}
#endif
@@ -132,7 +132,7 @@
void enableMiniMode()
{
if (!DebugHeap::tryGet())
- PerProcess<Scavenger>::get()->enableMiniMode();
+ Scavenger::get()->enableMiniMode();
}
} } // namespace bmalloc::api
Modified: trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj (242937 => 242938)
--- trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2019-03-14 08:02:00 UTC (rev 242938)
@@ -143,6 +143,7 @@
AD14AD29202529C400890E3B /* ProcessCheck.h in Headers */ = {isa = PBXBuildFile; fileRef = AD14AD27202529A600890E3B /* ProcessCheck.h */; };
AD14AD2A202529C700890E3B /* ProcessCheck.mm in Sources */ = {isa = PBXBuildFile; fileRef = AD14AD28202529B000890E3B /* ProcessCheck.mm */; };
DE8B13B321CC5D9F00A63FCD /* BVMTags.h in Headers */ = {isa = PBXBuildFile; fileRef = DE8B13B221CC5D9F00A63FCD /* BVMTags.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ E31E74802238CA5C005D084A /* StaticPerProcess.h in Headers */ = {isa = PBXBuildFile; fileRef = E31E747F2238CA5B005D084A /* StaticPerProcess.h */; settings = {ATTRIBUTES = (Private, ); }; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@@ -309,6 +310,7 @@
AD14AD27202529A600890E3B /* ProcessCheck.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ProcessCheck.h; path = bmalloc/ProcessCheck.h; sourceTree = "<group>"; };
AD14AD28202529B000890E3B /* ProcessCheck.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = ProcessCheck.mm; path = bmalloc/ProcessCheck.mm; sourceTree = "<group>"; };
DE8B13B221CC5D9F00A63FCD /* BVMTags.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = BVMTags.h; path = bmalloc/BVMTags.h; sourceTree = "<group>"; };
+ E31E747F2238CA5B005D084A /* StaticPerProcess.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = StaticPerProcess.h; path = bmalloc/StaticPerProcess.h; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@@ -552,6 +554,7 @@
144469FD17A61F1F00F9EA1D /* PerThread.h */,
145F6878179E3A4400D65598 /* Range.h */,
148EFAE61D6B953B008E721E /* ScopeExit.h */,
+ E31E747F2238CA5B005D084A /* StaticPerProcess.h */,
7C571F0022388B840077A3C7 /* StdLibExtras.h */,
1417F64F18B7280C0076FA3F /* Syscall.h */,
1479E21217A1A255006D4E9D /* Vector.h */,
@@ -665,6 +668,7 @@
14DD789018F48CEB00950702 /* Sizes.h in Headers */,
14DD78BC18F48D6B00950702 /* SmallLine.h in Headers */,
14DD78BD18F48D6B00950702 /* SmallPage.h in Headers */,
+ E31E74802238CA5C005D084A /* StaticPerProcess.h in Headers */,
7C571F0122388B840077A3C7 /* StdLibExtras.h in Headers */,
14DD78CE18F48D7500950702 /* Syscall.h in Headers */,
14DD78CF18F48D7500950702 /* Vector.h in Headers */,
Modified: trunk/Source/bmalloc/test/testbmalloc.cpp (242937 => 242938)
--- trunk/Source/bmalloc/test/testbmalloc.cpp 2019-03-14 08:00:56 UTC (rev 242937)
+++ trunk/Source/bmalloc/test/testbmalloc.cpp 2019-03-14 08:02:00 UTC (rev 242938)
@@ -73,7 +73,7 @@
static void assertEmptyPointerSet(const std::set<void*>& pointers)
{
- if (PerProcess<Environment>::get()->isDebugHeapEnabled()) {
+ if (Environment::get()->isDebugHeapEnabled()) {
printf(" skipping checks because DebugHeap.\n");
return;
}
@@ -90,7 +90,7 @@
template<typename heapType>
static void assertHasObjects(IsoHeap<heapType>& heap, std::set<void*> pointers)
{
- if (PerProcess<Environment>::get()->isDebugHeapEnabled()) {
+ if (Environment::get()->isDebugHeapEnabled()) {
printf(" skipping checks because DebugHeap.\n");
return;
}
@@ -106,7 +106,7 @@
template<typename heapType>
static void assertHasOnlyObjects(IsoHeap<heapType>& heap, std::set<void*> pointers)
{
- if (PerProcess<Environment>::get()->isDebugHeapEnabled()) {
+ if (Environment::get()->isDebugHeapEnabled()) {
printf(" skipping checks because DebugHeap.\n");
return;
}
@@ -123,7 +123,7 @@
static void assertClean(IsoHeap<heapType>& heap)
{
scavengeThisThread();
- if (!PerProcess<Environment>::get()->isDebugHeapEnabled()) {
+ if (!Environment::get()->isDebugHeapEnabled()) {
auto& impl = heap.impl();
{
std::lock_guard<Mutex> locker(impl.lock);
@@ -131,7 +131,7 @@
}
}
heap.scavenge();
- if (!PerProcess<Environment>::get()->isDebugHeapEnabled()) {
+ if (!Environment::get()->isDebugHeapEnabled()) {
auto& impl = heap.impl();
std::lock_guard<Mutex> locker(impl.lock);
CHECK(!impl.numCommittedPages());
