source: webkit/trunk/Source/JavaScriptCore/jit/ExecutableAllocator.cpp

Last change on this file was 295680, checked in by [email protected], 3 years ago

Enhance RawPointer to take function pointers as well.
​https://bugs.webkit.org/show_bug.cgi?id=241773

Reviewed by Yusuke Suzuki.

Also add PageBlock::end and PageReservation::end methods to make code a little
more readable (motivated by a RawPointer use).

  • Source/JavaScriptCore/API/JSMarkingConstraintPrivate.cpp:

(JSContextGroupAddMarkingConstraint):

  • Source/JavaScriptCore/heap/HeapFinalizerCallback.cpp:

(JSC::HeapFinalizerCallback::dump const):

  • Source/JavaScriptCore/jit/ExecutableAllocator.cpp:

(JSC::initializeJITPageReservation):

  • Source/WTF/wtf/PageBlock.h:

(WTF::PageBlock::end const):

  • Source/WTF/wtf/PageReservation.h:
  • Source/WTF/wtf/RawPointer.h:

(WTF::RawPointer::RawPointer):

Canonical link: ​https://commits.webkit.org/251685@main
File size: 48.0 KB
Line 
1/*
2 * Copyright (C) 2008-2022 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "ExecutableAllocator.h"
28
29#if ENABLE(JIT)
30
31#include "ExecutableAllocationFuzz.h"
32#include "JITOperationValidation.h"
33#include "LinkBuffer.h"
34#include <wtf/FastBitVector.h>
35#include <wtf/FileSystem.h>
36#include <wtf/FixedVector.h>
37#include <wtf/IterationStatus.h>
38#include <wtf/PageReservation.h>
39#include <wtf/ProcessID.h>
40#include <wtf/RedBlackTree.h>
41#include <wtf/Scope.h>
42#include <wtf/SystemTracing.h>
43#include <wtf/WorkQueue.h>
44
45#if USE(LIBPAS_JIT_HEAP)
46#include <bmalloc/jit_heap.h>
47#include <bmalloc/jit_heap_config.h>
48#else
49#include <wtf/MetaAllocator.h>
50#endif
51
52#if HAVE(IOS_JIT_RESTRICTIONS)
53#include <wtf/cocoa/Entitlements.h>
54#endif
55
56#if OS(DARWIN)
57#include <fcntl.h>
58#include <mach/mach.h>
59#include <mach/mach_time.h>
60
61extern "C" {
62 /* Routine mach_vm_remap */
63#ifdef mig_external
64 mig_external
65#else
66 extern
67#endif /* mig_external */
68 kern_return_t mach_vm_remap
69 (
70 vm_map_t target_task,
71 mach_vm_address_t *target_address,
72 mach_vm_size_t size,
73 mach_vm_offset_t mask,
74 int flags,
75 vm_map_t src_task,
76 mach_vm_address_t src_address,
77 boolean_t copy,
78 vm_prot_t *cur_protection,
79 vm_prot_t *max_protection,
80 vm_inherit_t inheritance
81 );
82}
83#endif
84
85namespace JSC {
86
87using namespace WTF;
88
89#if USE(LIBPAS_JIT_HEAP)
90static constexpr size_t minimumPoolSizeForSegregatedHeap = 256 * MB;
91#endif
92
93#if defined(FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB) && FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB > 0
94static constexpr size_t fixedExecutableMemoryPoolSize = FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB * MB;
95#elif CPU(ARM64)
96#if ENABLE(JUMP_ISLANDS)
97static constexpr size_t fixedExecutableMemoryPoolSize = 512 * MB;
98#else
99static constexpr size_t fixedExecutableMemoryPoolSize = 128 * MB;
100#endif
101#elif CPU(ARM_THUMB2)
102#if ENABLE(JUMP_ISLANDS)
103static constexpr size_t fixedExecutableMemoryPoolSize = 32 * MB;
104#else
105static constexpr size_t fixedExecutableMemoryPoolSize = 16 * MB;
106#endif
107#elif CPU(X86_64)
108static constexpr size_t fixedExecutableMemoryPoolSize = 1 * GB;
109#else
110static constexpr size_t fixedExecutableMemoryPoolSize = 32 * MB;
111#endif
112
113#if ENABLE(JUMP_ISLANDS)
114#if CPU(ARM64)
115static constexpr double islandRegionSizeFraction = 0.125;
116static constexpr size_t islandSizeInBytes = 4;
117#elif CPU(ARM_THUMB2)
118static constexpr double islandRegionSizeFraction = 0.05;
119static constexpr size_t islandSizeInBytes = 4;
120#endif
121#endif
122
123// Quick sanity check, in case FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB was set.
124#if !ENABLE(JUMP_ISLANDS)
125static_assert(fixedExecutableMemoryPoolSize <= MacroAssembler::nearJumpRange, "Executable pool size is too large for near jump/call without JUMP_ISLANDS");
126#endif
127
128#if CPU(ARM)
129static constexpr double executablePoolReservationFraction = 0.15;
130#else
131static constexpr double executablePoolReservationFraction = 0.25;
132#endif
133
134#if USE(LIBPAS_JIT_HEAP)
135// This size is derived from jit_config's medium table size.
136static constexpr size_t minimumExecutablePoolReservationSize = 256 * KB;
137static_assert(fixedExecutableMemoryPoolSize * executablePoolReservationFraction >= minimumExecutablePoolReservationSize);
138static_assert(fixedExecutableMemoryPoolSize < 4 * GB, "ExecutableMemoryHandle assumes it is less than 4GB");
139#endif
140
141static bool isJITEnabled()
142{
143 bool jitEnabled = !g_jscConfig.jitDisabled;
144#if HAVE(IOS_JIT_RESTRICTIONS)
145 return processHasEntitlement("dynamic-codesigning"_s) && jitEnabled;
146#else
147 return jitEnabled;
148#endif
149}
150
151void ExecutableAllocator::setJITEnabled(bool enabled)
152{
153 bool jitEnabled = !g_jscConfig.jitDisabled;
154 ASSERT(!g_jscConfig.fixedVMPoolExecutableAllocator);
155 if (jitEnabled == enabled)
156 return;
157
158 g_jscConfig.jitDisabled = !enabled;
159
160#if HAVE(IOS_JIT_RESTRICTIONS)
161 if (!enabled && processHasEntitlement("dynamic-codesigning"_s)) {
162 // Because of an OS quirk, even after the JIT region has been unmapped,
163 // the OS thinks that region is reserved, and as such, can cause Gigacage
164 // allocation to fail. We work around this by initializing the Gigacage
165 // first.
166 // Note: when called, setJITEnabled() is always called extra early in the
167 // process bootstrap. Under normal operation (when setJITEnabled() isn't
168 // called at all), we will naturally initialize the Gigacage before we
169 // allocate the JIT region. Hence, this workaround is merely ensuring the
170 // same behavior of allocation ordering.
171 Gigacage::ensureGigacage();
172
173 constexpr size_t size = 1;
174 constexpr int protection = PROT_READ | PROT_WRITE | PROT_EXEC;
175 constexpr int fd = OSAllocator::JSJITCodePages;
176 int flags = MAP_PRIVATE | MAP_ANON | (Options::useJITCage() ? MAP_EXECUTABLE_FOR_JIT_WITH_JIT_CAGE : MAP_EXECUTABLE_FOR_JIT);
177 void* allocation = mmap(nullptr, size, protection, flags, fd, 0);
178 const void* executableMemoryAllocationFailure = reinterpret_cast<void*>(-1);
179 RELEASE_ASSERT_WITH_MESSAGE(allocation && allocation != executableMemoryAllocationFailure, "We should not have allocated executable memory before disabling the JIT.");
180 RELEASE_ASSERT_WITH_MESSAGE(!munmap(allocation, size), "Unmapping executable memory should succeed so we do not have any executable memory in the address space");
181 RELEASE_ASSERT_WITH_MESSAGE(mmap(nullptr, size, protection, flags, fd, 0) == executableMemoryAllocationFailure, "Allocating executable memory should fail after setJITEnabled(false) is called.");
182 }
183#endif
184}
185
186#if OS(DARWIN) && HAVE(REMAP_JIT)
187
188#if USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)
189static ALWAYS_INLINE MacroAssemblerCodeRef<JITThunkPtrTag> jitWriteThunkGenerator(void* writableAddr, void* stubBase, size_t stubSize)
190{
191 auto exitScope = makeScopeExit([] {
192 RELEASE_ASSERT(!g_jscConfig.useFastJITPermissions);
193 });
194
195 using namespace ARM64Registers;
196 using TrustedImm32 = MacroAssembler::TrustedImm32;
197
198 MacroAssembler jit;
199
200 jit.tagReturnAddress();
201 jit.move(MacroAssembler::TrustedImmPtr(writableAddr), x7);
202 jit.addPtr(x7, x0);
203
204 jit.move(x0, x3);
205 MacroAssembler::Jump smallCopy = jit.branch64(MacroAssembler::Below, x2, MacroAssembler::TrustedImm64(64));
206
207 jit.add64(TrustedImm32(32), x3);
208 jit.and64(TrustedImm32(-32), x3);
209 jit.loadPair64(x1, x12, x13);
210 jit.loadPair64(x1, TrustedImm32(16), x14, x15);
211 jit.sub64(x3, x0, x5);
212 jit.addPtr(x5, x1);
213
214 jit.loadPair64(x1, x8, x9);
215 jit.loadPair64(x1, TrustedImm32(16), x10, x11);
216 jit.add64(TrustedImm32(32), x1);
217 jit.sub64(x5, x2);
218 jit.storePair64(x12, x13, x0);
219 jit.storePair64(x14, x15, x0, TrustedImm32(16));
220 MacroAssembler::Jump cleanup = jit.branchSub64(MacroAssembler::BelowOrEqual, TrustedImm32(64), x2);
221
222 MacroAssembler::Label copyLoop = jit.label();
223 jit.storePair64WithNonTemporalAccess(x8, x9, x3);
224 jit.storePair64WithNonTemporalAccess(x10, x11, x3, TrustedImm32(16));
225 jit.add64(TrustedImm32(32), x3);
226 jit.loadPair64WithNonTemporalAccess(x1, x8, x9);
227 jit.loadPair64WithNonTemporalAccess(x1, TrustedImm32(16), x10, x11);
228 jit.add64(TrustedImm32(32), x1);
229 jit.branchSub64(MacroAssembler::Above, TrustedImm32(32), x2).linkTo(copyLoop, &jit);
230
231 cleanup.link(&jit);
232 jit.add64(x2, x1);
233 jit.loadPair64(x1, x12, x13);
234 jit.loadPair64(x1, TrustedImm32(16), x14, x15);
235 jit.storePair64(x8, x9, x3);
236 jit.storePair64(x10, x11, x3, TrustedImm32(16));
237 jit.addPtr(x2, x3);
238 jit.storePair64(x12, x13, x3, TrustedImm32(32));
239 jit.storePair64(x14, x15, x3, TrustedImm32(48));
240 jit.ret();
241
242 MacroAssembler::Label local0 = jit.label();
243 jit.load64(MacroAssembler::PostIndexAddress(x1, 8), x6);
244 jit.store64(x6, MacroAssembler::PostIndexAddress(x3, 8));
245 smallCopy.link(&jit);
246 jit.branchSub64(MacroAssembler::AboveOrEqual, TrustedImm32(8), x2).linkTo(local0, &jit);
247 MacroAssembler::Jump local2 = jit.branchAdd64(MacroAssembler::Equal, TrustedImm32(8), x2);
248 MacroAssembler::Label local1 = jit.label();
249 jit.load8(x1, PostIndex(1), x6);
250 jit.store8(x6, x3, PostIndex(1));
251 jit.branchSub64(MacroAssembler::NotEqual, TrustedImm32(1), x2).linkTo(local1, &jit);
252 local2.link(&jit);
253 jit.ret();
254
255 auto stubBaseCodePtr = MacroAssemblerCodePtr<LinkBufferPtrTag>(tagCodePtr<LinkBufferPtrTag>(stubBase));
256 LinkBuffer linkBuffer(jit, stubBaseCodePtr, stubSize, LinkBuffer::Profile::Thunk);
257 // We don't use FINALIZE_CODE() for two reasons.
258 // The first is that we don't want the writeable address, as disassembled instructions,
259 // to appear in the console or anywhere in memory, via the PrintStream buffer.
260 // The second is we can't guarantee that the code is readable when using the
261 // asyncDisassembly option as our caller will set our pages execute only.
262 return linkBuffer.finalizeCodeWithoutDisassembly<JITThunkPtrTag>();
263}
264#else // not USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)
265static void genericWriteToJITRegion(off_t offset, const void* data, size_t dataSize)
266{
267 memcpy((void*)(g_jscConfig.startOfFixedWritableMemoryPool + offset), data, dataSize);
268}
269
270static MacroAssemblerCodeRef<JITThunkPtrTag> ALWAYS_INLINE jitWriteThunkGenerator(void* address, void*, size_t)
271{
272 g_jscConfig.startOfFixedWritableMemoryPool = reinterpret_cast<uintptr_t>(address);
273 void* function = reinterpret_cast<void*>(&genericWriteToJITRegion);
274#if CPU(ARM_THUMB2)
275 // Handle thumb offset
276 uintptr_t functionAsInt = reinterpret_cast<uintptr_t>(function);
277 functionAsInt -= 1;
278 function = reinterpret_cast<void*>(functionAsInt);
279#endif
280 auto codePtr = MacroAssemblerCodePtr<JITThunkPtrTag>(tagCFunctionPtr<JITThunkPtrTag>(function));
281 return MacroAssemblerCodeRef<JITThunkPtrTag>::createSelfManagedCodeRef(codePtr);
282}
283#endif // USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)
284
285static ALWAYS_INLINE void initializeSeparatedWXHeaps(void* stubBase, size_t stubSize, void* jitBase, size_t jitSize)
286{
287 auto exitScope = makeScopeExit([] {
288 RELEASE_ASSERT(!g_jscConfig.useFastJITPermissions);
289 });
290
291 mach_vm_address_t writableAddr = 0;
292
293 // Create a second mapping of the JIT region at a random address.
294 vm_prot_t cur, max;
295 int remapFlags = VM_FLAGS_ANYWHERE;
296#if defined(VM_FLAGS_RANDOM_ADDR)
297 remapFlags |= VM_FLAGS_RANDOM_ADDR;
298#endif
299 kern_return_t ret = mach_vm_remap(mach_task_self(), &writableAddr, jitSize, 0,
300 remapFlags,
301 mach_task_self(), (mach_vm_address_t)jitBase, FALSE,
302 &cur, &max, VM_INHERIT_DEFAULT);
303
304 bool remapSucceeded = (ret == KERN_SUCCESS);
305 if (!remapSucceeded)
306 return;
307
308 // Assemble a thunk that will serve as the means for writing into the JIT region.
309 MacroAssemblerCodeRef<JITThunkPtrTag> writeThunk = jitWriteThunkGenerator(reinterpret_cast<void*>(writableAddr), stubBase, stubSize);
310
311 int result = 0;
312
313#if USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)
314 // Prevent reading the write thunk code.
315 result = vm_protect(mach_task_self(), reinterpret_cast<vm_address_t>(stubBase), stubSize, true, VM_PROT_EXECUTE);
316 RELEASE_ASSERT(!result);
317#endif
318
319 // Prevent writing into the executable JIT mapping.
320 result = vm_protect(mach_task_self(), reinterpret_cast<vm_address_t>(jitBase), jitSize, true, VM_PROT_READ | VM_PROT_EXECUTE);
321 RELEASE_ASSERT(!result);
322
323 // Prevent execution in the writable JIT mapping.
324 result = vm_protect(mach_task_self(), static_cast<vm_address_t>(writableAddr), jitSize, true, VM_PROT_READ | VM_PROT_WRITE);
325 RELEASE_ASSERT(!result);
326
327 // Zero out writableAddr to avoid leaking the address of the writable mapping.
328 memset_s(&writableAddr, sizeof(writableAddr), 0, sizeof(writableAddr));
329
330#if ENABLE(SEPARATED_WX_HEAP)
331 g_jscConfig.jitWriteSeparateHeaps = reinterpret_cast<JITWriteSeparateHeapsFunction>(writeThunk.code().executableAddress());
332#endif
333}
334
335#else // OS(DARWIN) && HAVE(REMAP_JIT)
336static ALWAYS_INLINE void initializeSeparatedWXHeaps(void*, size_t, void*, size_t)
337{
338}
339#endif
340
341struct JITReservation {
342 PageReservation pageReservation;
343 void* base { nullptr };
344 size_t size { 0 };
345};
346
347static ALWAYS_INLINE JITReservation initializeJITPageReservation()
348{
349 JITReservation reservation;
350 if (!isJITEnabled())
351 return reservation;
352
353 reservation.size = fixedExecutableMemoryPoolSize;
354
355 if (Options::jitMemoryReservationSize()) {
356 reservation.size = Options::jitMemoryReservationSize();
357#if USE(LIBPAS_JIT_HEAP)
358 if (reservation.size * executablePoolReservationFraction < minimumExecutablePoolReservationSize)
359 reservation.size += minimumExecutablePoolReservationSize;
360#endif
361 }
362 reservation.size = std::max(roundUpToMultipleOf(pageSize(), reservation.size), pageSize() * 2);
363
364#if !ENABLE(JUMP_ISLANDS)
365 RELEASE_ASSERT(reservation.size <= MacroAssembler::nearJumpRange, "Executable pool size is too large for near jump/call without JUMP_ISLANDS");
366#endif
367
368#if USE(LIBPAS_JIT_HEAP)
369 if (reservation.size < minimumPoolSizeForSegregatedHeap)
370 jit_heap_runtime_config.max_segregated_object_size = 0;
371#endif
372
373 auto tryCreatePageReservation = [] (size_t reservationSize) {
374#if OS(LINUX)
375 // If we use uncommitted reservation, mmap operation is recorded with small page size in perf command's output.
376 // This makes the following JIT code logging broken and some of JIT code is not recorded correctly.
377 // To avoid this problem, we use committed reservation if we need perf JITDump logging.
378 if (Options::logJITCodeForPerf())
379 return PageReservation::tryReserveAndCommitWithGuardPages(reservationSize, OSAllocator::JSJITCodePages, EXECUTABLE_POOL_WRITABLE, true, false);
380#endif
381 if (Options::useJITCage() && JSC_ALLOW_JIT_CAGE_SPECIFIC_RESERVATION)
382 return PageReservation::tryReserve(reservationSize, OSAllocator::JSJITCodePages, EXECUTABLE_POOL_WRITABLE, true, Options::useJITCage());
383 return PageReservation::tryReserveWithGuardPages(reservationSize, OSAllocator::JSJITCodePages, EXECUTABLE_POOL_WRITABLE, true, false);
384 };
385
386 reservation.pageReservation = tryCreatePageReservation(reservation.size);
387
388 if (Options::verboseExecutablePoolAllocation())
389 dataLog(getpid(), ": Got executable pool reservation at ", RawPointer(reservation.pageReservation.base()), "...", RawPointer(reservation.pageReservation.end()), ", while I'm at ", RawPointer(bitwise_cast<void*>(initializeJITPageReservation)), "\n");
390
391 if (reservation.pageReservation) {
392 ASSERT(reservation.pageReservation.size() == reservation.size);
393 reservation.base = reservation.pageReservation.base();
394
395 bool fastJITPermissionsIsSupported = false;
396#if OS(DARWIN) && CPU(ARM64)
397#if USE(PTHREAD_JIT_PERMISSIONS_API)
398 fastJITPermissionsIsSupported = !!pthread_jit_write_protect_supported_np();
399#elif USE(APPLE_INTERNAL_SDK)
400 fastJITPermissionsIsSupported = !!os_thread_self_restrict_rwx_is_supported();
401#endif
402#endif
403 g_jscConfig.useFastJITPermissions = fastJITPermissionsIsSupported;
404
405 if (g_jscConfig.useFastJITPermissions)
406 threadSelfRestrictRWXToRX();
407
408#if ENABLE(SEPARATED_WX_HEAP)
409 if (!g_jscConfig.useFastJITPermissions) {
410 // First page of our JIT allocation is reserved.
411 ASSERT(reservation.size >= pageSize() * 2);
412 reservation.base = (void*)((uintptr_t)(reservation.base) + pageSize());
413 reservation.size -= pageSize();
414 initializeSeparatedWXHeaps(reservation.pageReservation.base(), pageSize(), reservation.base, reservation.size);
415 }
416#endif
417
418 void* reservationEnd = reinterpret_cast<uint8_t*>(reservation.base) + reservation.size;
419 g_jscConfig.startExecutableMemory = reservation.base;
420 g_jscConfig.endExecutableMemory = reservationEnd;
421
422#if !USE(SYSTEM_MALLOC) && ENABLE(UNIFIED_AND_FREEZABLE_CONFIG_RECORD)
423 WebConfig::g_config[0] = bitwise_cast<uintptr_t>(reservation.base);
424 WebConfig::g_config[1] = bitwise_cast<uintptr_t>(reservationEnd);
425#endif
426 }
427
428 return reservation;
429}
430
431class FixedVMPoolExecutableAllocator final {
432 WTF_MAKE_FAST_ALLOCATED;
433
434#if ENABLE(JUMP_ISLANDS)
435 class Islands;
436 class RegionAllocator;
437#endif
438
439public:
440 FixedVMPoolExecutableAllocator()
441#if !ENABLE(JUMP_ISLANDS)
442 : m_allocator(*this)
443#endif
444 {
445 JITReservation reservation = initializeJITPageReservation();
446 m_reservation = WTFMove(reservation.pageReservation);
447 if (m_reservation) {
448#if ENABLE(JUMP_ISLANDS)
449 // These sizes guarantee that any jump within an island can jump forwards or backwards
450 // to the adjacent island in a single instruction.
451 const size_t islandRegionSize = roundUpToMultipleOf(pageSize(), static_cast<size_t>(MacroAssembler::nearJumpRange * islandRegionSizeFraction));
452 m_regionSize = MacroAssembler::nearJumpRange - islandRegionSize;
453 RELEASE_ASSERT(isPageAligned(islandRegionSize));
454 RELEASE_ASSERT(isPageAligned(m_regionSize));
455 const unsigned numAllocators = (reservation.size + m_regionSize - 1) / m_regionSize;
456 m_allocators = FixedVector<RegionAllocator>::createWithSizeAndConstructorArguments(numAllocators, *this);
457
458 uintptr_t start = bitwise_cast<uintptr_t>(memoryStart());
459 uintptr_t reservationEnd = bitwise_cast<uintptr_t>(memoryEnd());
460 for (size_t i = 0; i < numAllocators; ++i) {
461 uintptr_t end = start + m_regionSize;
462 uintptr_t islandBegin = end - islandRegionSize;
463 // The island in the very last region is never actually used (everything goes backwards), but we
464 // can't put code there in case they do need to use a backward jump island, so set up accordingly.
465 if (i == numAllocators - 1)
466 islandBegin = end = std::min(islandBegin, reservationEnd);
467 RELEASE_ASSERT(end <= reservationEnd);
468 m_allocators[i].configure(start, islandBegin, end);
469 m_bytesReserved += m_allocators[i].allocatorSize();
470 start += m_regionSize;
471 }
472#else
473 m_allocator.addFreshFreeSpace(reservation.base, reservation.size);
474 m_bytesReserved += reservation.size;
475#endif
476 }
477 }
478
479 ~FixedVMPoolExecutableAllocator()
480 {
481 m_reservation.deallocate();
482 }
483
484 void* memoryStart() { return g_jscConfig.startExecutableMemory; }
485 void* memoryEnd() { return g_jscConfig.endExecutableMemory; }
486 bool isValid() { return !!m_reservation; }
487
488 RefPtr<ExecutableMemoryHandle> allocate(size_t sizeInBytes)
489 {
490#if USE(LIBPAS_JIT_HEAP)
491 auto result = ExecutableMemoryHandle::createImpl(sizeInBytes);
492 if (LIKELY(result))
493 m_bytesAllocated.fetch_add(result->sizeInBytes(), std::memory_order_relaxed);
494 return result;
495#elif ENABLE(JUMP_ISLANDS)
496 Locker locker { getLock() };
497
498 unsigned start = 0;
499 if (Options::useRandomizingExecutableIslandAllocation())
500 start = cryptographicallyRandomNumber() % m_allocators.size();
501
502 unsigned i = start;
503 while (true) {
504 RegionAllocator& allocator = m_allocators[i];
505 if (RefPtr<ExecutableMemoryHandle> result = allocator.allocate(locker, sizeInBytes))
506 return result;
507 i = (i + 1) % m_allocators.size();
508 if (i == start)
509 break;
510 }
511 return nullptr;
512#else
513 return m_allocator.allocate(sizeInBytes);
514#endif // ENABLE(JUMP_ISLANDS)
515 }
516
517 Lock& getLock() WTF_RETURNS_LOCK(m_lock) { return m_lock; }
518
519#if USE(LIBPAS_JIT_HEAP)
520 void shrinkBytesAllocated(size_t oldSizeInBytes, size_t newSizeInBytes)
521 {
522 m_bytesAllocated.fetch_add(newSizeInBytes - oldSizeInBytes, std::memory_order_relaxed);
523 }
524#endif
525
526 // Non atomic
527 size_t bytesAllocated()
528 {
529#if USE(LIBPAS_JIT_HEAP)
530 return m_bytesAllocated.load(std::memory_order_relaxed);
531#else
532 size_t result = 0;
533 forEachAllocator([&] (Allocator& allocator) {
534 result += allocator.bytesAllocated();
535 });
536 return result;
537#endif
538 }
539
540 size_t bytesReserved() const
541 {
542 return m_bytesReserved;
543 }
544
545 size_t bytesAvailable()
546 {
547 size_t bytesReserved = this->bytesReserved();
548#if USE(LIBPAS_JIT_HEAP)
549 size_t nonAvailableSize = static_cast<size_t>(bytesReserved * executablePoolReservationFraction);
550 if (nonAvailableSize < minimumExecutablePoolReservationSize)
551 return bytesReserved - minimumExecutablePoolReservationSize;
552 return bytesReserved - nonAvailableSize;
553#else
554 return static_cast<size_t>(bytesReserved * (1 - executablePoolReservationFraction));
555#endif
556 }
557
558#if !USE(LIBPAS_JIT_HEAP)
559 size_t bytesCommitted()
560 {
561 size_t result = 0;
562 forEachAllocator([&] (Allocator& allocator) {
563 result += allocator.bytesCommitted();
564 });
565 return result;
566 }
567#endif
568
569 bool isInAllocatedMemory(const AbstractLocker& locker, void* address)
570 {
571#if ENABLE(JUMP_ISLANDS)
572 if (RegionAllocator* allocator = findRegion(bitwise_cast<uintptr_t>(address)))
573 return allocator->isInAllocatedMemory(locker, address);
574 return false;
575#else
576 return m_allocator.isInAllocatedMemory(locker, address);
577#endif
578 }
579
580#if ENABLE(META_ALLOCATOR_PROFILE)
581 void dumpProfile()
582 {
583 forEachAllocator([&] (Allocator& allocator) {
584 allocator.dumpProfile();
585 });
586 }
587#endif
588
589#if !USE(LIBPAS_JIT_HEAP)
590 MetaAllocator::Statistics currentStatistics()
591 {
592 Locker locker { getLock() };
593 MetaAllocator::Statistics result { 0, 0, 0 };
594 forEachAllocator([&] (Allocator& allocator) {
595 auto allocatorStats = allocator.currentStatistics(locker);
596 result.bytesAllocated += allocatorStats.bytesAllocated;
597 result.bytesReserved += allocatorStats.bytesReserved;
598 result.bytesCommitted += allocatorStats.bytesCommitted;
599 });
600 return result;
601 }
602#endif // !USE(LIBPAS_JIT_HEAP)
603
604#if USE(LIBPAS_JIT_HEAP)
605 void handleWillBeReleased(ExecutableMemoryHandle& handle, size_t sizeInBytes)
606 {
607 m_bytesAllocated.fetch_sub(sizeInBytes, std::memory_order_relaxed);
608#if ENABLE(JUMP_ISLANDS)
609 if (m_islandsForJumpSourceLocation.isEmpty())
610 return;
611
612 Locker locker { getLock() };
613 handleWillBeReleased(locker, handle);
614#else // ENABLE(JUMP_ISLANDS) -> so !ENABLE(JUMP_ISLANDS)
615 UNUSED_PARAM(handle);
616#endif // ENABLE(JUMP_ISLANDS) -> so end of !ENABLE(JUMP_ISLANDS)
617 }
618#endif // USE(LIBPAS_JIT_HEAP)
619
620#if ENABLE(JUMP_ISLANDS)
621 void handleWillBeReleased(const LockHolder& locker, ExecutableMemoryHandle& handle)
622 {
623 if (m_islandsForJumpSourceLocation.isEmpty())
624 return;
625
626 Vector<Islands*, 16> toRemove;
627 void* start = handle.start().untaggedPtr();
628 void* end = handle.end().untaggedPtr();
629 m_islandsForJumpSourceLocation.iterate([&] (Islands& islands, bool& visitLeft, bool& visitRight) {
630 if (start <= islands.key() && islands.key() < end)
631 toRemove.append(&islands);
632 if (islands.key() > start)
633 visitLeft = true;
634 if (islands.key() < end)
635 visitRight = true;
636 });
637
638 for (Islands* islands : toRemove)
639 freeIslands(locker, islands);
640
641 if (ASSERT_ENABLED) {
642 m_islandsForJumpSourceLocation.iterate([&] (Islands& islands, bool& visitLeft, bool& visitRight) {
643 if (start <= islands.key() && islands.key() < end) {
644 dataLogLn("did not remove everything!");
645 RELEASE_ASSERT_NOT_REACHED();
646 }
647 visitLeft = true;
648 visitRight = true;
649 });
650 }
651 }
652
653 void* makeIsland(uintptr_t jumpLocation, uintptr_t newTarget, bool concurrently)
654 {
655 Locker locker { getLock() };
656 return islandForJumpLocation(locker, jumpLocation, newTarget, concurrently);
657 }
658
659private:
660 RegionAllocator* findRegion(uintptr_t ptr)
661 {
662 RegionAllocator* result = nullptr;
663 forEachAllocator([&] (RegionAllocator& allocator) {
664 if (allocator.start() <= ptr && ptr < allocator.end()) {
665 result = &allocator;
666 return IterationStatus::Done;
667 }
668 return IterationStatus::Continue;
669 });
670 return result;
671 }
672
673 void freeJumpIslands(const LockHolder&, Islands* islands)
674 {
675 for (CodeLocationLabel<ExecutableMemoryPtrTag> jumpIsland : islands->jumpIslands) {
676 uintptr_t untaggedJumpIsland = bitwise_cast<uintptr_t>(jumpIsland.dataLocation());
677 RegionAllocator* allocator = findRegion(untaggedJumpIsland);
678 RELEASE_ASSERT(allocator);
679 allocator->freeIsland(untaggedJumpIsland);
680 }
681 islands->jumpIslands.clear();
682 }
683
684 void freeIslands(const LockHolder& locker, Islands* islands)
685 {
686 freeJumpIslands(locker, islands);
687 m_islandsForJumpSourceLocation.remove(islands);
688 delete islands;
689 }
690
691 void* islandForJumpLocation(const LockHolder& locker, uintptr_t jumpLocation, uintptr_t target, bool concurrently)
692 {
693 Islands* islands = m_islandsForJumpSourceLocation.findExact(bitwise_cast<void*>(jumpLocation));
694 if (islands) {
695 // FIXME: We could create some method of reusing already allocated islands here, but it's
696 // unlikely to matter in practice.
697 if (!concurrently)
698 freeJumpIslands(locker, islands);
699 } else {
700 islands = new Islands;
701 islands->jumpSourceLocation = CodeLocationLabel<ExecutableMemoryPtrTag>(tagCodePtr<ExecutableMemoryPtrTag>(bitwise_cast<void*>(jumpLocation)));
702 m_islandsForJumpSourceLocation.insert(islands);
703 }
704
705 RegionAllocator* allocator = findRegion(jumpLocation > target ? jumpLocation - m_regionSize : jumpLocation);
706 RELEASE_ASSERT(allocator);
707 void* result = allocator->allocateIsland();
708 void* currentIsland = result;
709 jumpLocation = bitwise_cast<uintptr_t>(currentIsland);
710 while (true) {
711 islands->jumpIslands.append(CodeLocationLabel<ExecutableMemoryPtrTag>(tagCodePtr<ExecutableMemoryPtrTag>(currentIsland)));
712
713 auto emitJumpTo = [&] (void* target) {
714 RELEASE_ASSERT(Assembler::canEmitJump(bitwise_cast<void*>(jumpLocation), target));
715
716 MacroAssembler jit;
717 auto nearTailCall = jit.nearTailCall();
718 LinkBuffer linkBuffer(jit, MacroAssemblerCodePtr<NoPtrTag>(currentIsland), islandSizeInBytes, LinkBuffer::Profile::JumpIsland, JITCompilationMustSucceed, false);
719 RELEASE_ASSERT(linkBuffer.isValid());
720
721 // We use this to appease the assertion that we're not finalizing on a compiler thread. In this situation, it's
722 // ok to do this on a compiler thread, since the compiler thread is linking a jump to this code (and no other live
723 // code can jump to these islands). It's ok because the CPU protocol for exposing code to other CPUs is:
724 // - Self modifying code fence (what FINALIZE_CODE does below). This does various memory flushes + instruction sync barrier (isb).
725 // - Any CPU that will run the code must run a crossModifyingCodeFence (isb) before running it. Since the code that
726 // has a jump linked to this island hasn't finalized yet, they're guaranteed to finalize there code and run an isb.
727 linkBuffer.setIsJumpIsland();
728
729 linkBuffer.link(nearTailCall, CodeLocationLabel<NoPtrTag>(target));
730 FINALIZE_CODE(linkBuffer, NoPtrTag, "Jump Island: %lu", jumpLocation);
731 };
732
733 if (Assembler::canEmitJump(bitwise_cast<void*>(jumpLocation), bitwise_cast<void*>(target))) {
734 emitJumpTo(bitwise_cast<void*>(target));
735 break;
736 }
737
738 uintptr_t nextIslandRegion;
739 if (jumpLocation > target)
740 nextIslandRegion = jumpLocation - m_regionSize;
741 else
742 nextIslandRegion = jumpLocation + m_regionSize;
743
744 RegionAllocator* allocator = findRegion(nextIslandRegion);
745 RELEASE_ASSERT(allocator);
746 void* nextIsland = allocator->allocateIsland();
747 emitJumpTo(nextIsland);
748 jumpLocation = bitwise_cast<uintptr_t>(nextIsland);
749 currentIsland = nextIsland;
750 }
751
752 return result;
753 }
754#endif // ENABLE(JUMP_ISLANDS)
755
756private:
757 class Allocator
758#if !USE(LIBPAS_JIT_HEAP)
759 : public MetaAllocator
760#endif
761 {
762#if !USE(LIBPAS_JIT_HEAP)
763 using Base = MetaAllocator;
764#endif
765 public:
766 Allocator(FixedVMPoolExecutableAllocator& allocator)
767#if !USE(LIBPAS_JIT_HEAP)
768 : Base(allocator.getLock(), jitAllocationGranule, pageSize()) // round up all allocations to 32 bytes
769 ,
770#else
771 :
772#endif
773 m_fixedAllocator(allocator)
774 {
775 }
776
777#if USE(LIBPAS_JIT_HEAP)
778 void addFreshFreeSpace(void* start, size_t sizeInBytes)
779 {
780 RELEASE_ASSERT(!m_start);
781 RELEASE_ASSERT(!m_end);
782 m_start = reinterpret_cast<uintptr_t>(start);
783 m_end = m_start + sizeInBytes;
784 jit_heap_add_fresh_memory(pas_range_create(m_start, m_end));
785 }
786
787 bool isInAllocatedMemory(const AbstractLocker&, void* address)
788 {
789 uintptr_t addressAsInt = reinterpret_cast<uintptr_t>(address);
790 return addressAsInt >= m_start && addressAsInt < m_end;
791 }
792#endif // USE(LIBPAS_JIT_HEAP)
793
794#if !USE(LIBPAS_JIT_HEAP)
795 FreeSpacePtr allocateNewSpace(size_t&) override
796 {
797 // We're operating in a fixed pool, so new allocation is always prohibited.
798 return nullptr;
799 }
800
801 void notifyNeedPage(void* page, size_t count) override
802 {
803 m_fixedAllocator.m_reservation.commit(page, pageSize() * count);
804 }
805
806 void notifyPageIsFree(void* page, size_t count) override
807 {
808 m_fixedAllocator.m_reservation.decommit(page, pageSize() * count);
809 }
810#endif // !USE(LIBPAS_JIT_HEAP)
811
812 FixedVMPoolExecutableAllocator& m_fixedAllocator;
813#if USE(LIBPAS_JIT_HEAP)
814 uintptr_t m_start { 0 };
815 uintptr_t m_end { 0 };
816#endif // USE(LIBPAS_JIT_HEAP)
817 };
818
819#if ENABLE(JUMP_ISLANDS)
820 class RegionAllocator final : public Allocator {
821 using Base = Allocator;
822 public:
823 RegionAllocator(FixedVMPoolExecutableAllocator& allocator)
824 : Base(allocator)
825 {
826 RELEASE_ASSERT(!(pageSize() % islandSizeInBytes), "Current implementation relies on this");
827 }
828
829 void configure(uintptr_t start, uintptr_t islandBegin, uintptr_t end)
830 {
831 RELEASE_ASSERT(start < islandBegin);
832 RELEASE_ASSERT(islandBegin <= end);
833 m_start = tagCodePtr<ExecutableMemoryPtrTag>(bitwise_cast<void*>(start));
834 m_islandBegin = tagCodePtr<ExecutableMemoryPtrTag>(bitwise_cast<void*>(islandBegin));
835 m_end = tagCodePtr<ExecutableMemoryPtrTag>(bitwise_cast<void*>(end));
836 RELEASE_ASSERT(!((this->end() - this->start()) % pageSize()));
837 RELEASE_ASSERT(!((this->end() - this->islandBegin()) % pageSize()));
838 addFreshFreeSpace(bitwise_cast<void*>(this->start()), allocatorSize());
839 }
840
841 // ------------------------------------
842 // | jit allocations --> <-- islands |
843 // -------------------------------------
844
845 uintptr_t start() { return bitwise_cast<uintptr_t>(untagCodePtr<ExecutableMemoryPtrTag>(m_start)); }
846 uintptr_t islandBegin() { return bitwise_cast<uintptr_t>(untagCodePtr<ExecutableMemoryPtrTag>(m_islandBegin)); }
847 uintptr_t end() { return bitwise_cast<uintptr_t>(untagCodePtr<ExecutableMemoryPtrTag>(m_end)); }
848
849 size_t maxIslandsInThisRegion() { return (end() - islandBegin()) / islandSizeInBytes; }
850
851 uintptr_t allocatorSize()
852 {
853 return islandBegin() - start();
854 }
855
856 size_t islandsPerPage()
857 {
858 size_t islandsPerPage = pageSize() / islandSizeInBytes;
859 ASSERT(islandsPerPage * islandSizeInBytes == pageSize());
860 ASSERT(isPowerOfTwo(islandsPerPage));
861 return islandsPerPage;
862 }
863
864#if !USE(LIBPAS_JIT_HEAP)
865 void release(const LockHolder& locker, MetaAllocatorHandle& handle) final
866 {
867 m_fixedAllocator.handleWillBeReleased(locker, handle);
868 Base::release(locker, handle);
869 }
870#endif
871
872 void* allocateIsland()
873 {
874 uintptr_t end = this->end();
875 auto findResult = [&] () -> void* {
876 size_t resultBit = islandBits.findClearBit(0);
877 if (resultBit == islandBits.size())
878 return nullptr;
879 islandBits[resultBit] = true;
880 uintptr_t result = end - ((resultBit + 1) * islandSizeInBytes);
881 return bitwise_cast<void*>(result);
882 };
883
884 if (void* result = findResult())
885 return result;
886
887 const size_t oldSize = islandBits.size();
888 const size_t maxIslandsInThisRegion = this->maxIslandsInThisRegion();
889
890 RELEASE_ASSERT(oldSize <= maxIslandsInThisRegion);
891 if (UNLIKELY(oldSize == maxIslandsInThisRegion))
892 crashOnJumpIslandExhaustion();
893
894 const size_t newSize = std::min(oldSize + islandsPerPage(), maxIslandsInThisRegion);
895 islandBits.resize(newSize);
896
897 uintptr_t islandsBegin = end - (newSize * islandSizeInBytes); // [islandsBegin, end)
898 m_fixedAllocator.m_reservation.commit(bitwise_cast<void*>(islandsBegin), (newSize - oldSize) * islandSizeInBytes);
899
900 void* result = findResult();
901 RELEASE_ASSERT(result);
902 return result;
903 }
904
905 NEVER_INLINE NO_RETURN_DUE_TO_CRASH void crashOnJumpIslandExhaustion()
906 {
907 CRASH();
908 }
909
910 std::optional<size_t> islandBit(uintptr_t island)
911 {
912 uintptr_t end = this->end();
913 if (islandBegin() <= island && island < end)
914 return ((end - island) / islandSizeInBytes) - 1;
915 return std::nullopt;
916 }
917
918 void freeIsland(uintptr_t island)
919 {
920 RELEASE_ASSERT(islandBegin() <= island && island < end());
921 size_t bit = islandBit(island).value();
922 RELEASE_ASSERT(!!islandBits[bit]);
923 islandBits[bit] = false;
924 }
925
926 bool isInAllocatedMemory(const AbstractLocker& locker, void* address)
927 {
928 if (Base::isInAllocatedMemory(locker, address))
929 return true;
930 if (std::optional<size_t> bit = islandBit(bitwise_cast<uintptr_t>(address))) {
931 if (bit.value() < islandBits.size())
932 return !!islandBits[bit.value()];
933 }
934 return false;
935 }
936
937 private:
938 // Range: [start, end)
939 void* m_start;
940 void* m_islandBegin;
941 void* m_end;
942 FastBitVector islandBits;
943 };
944#endif // ENABLE(JUMP_ISLANDS)
945
946 template <typename Function>
947 void forEachAllocator(Function function)
948 {
949#if ENABLE(JUMP_ISLANDS)
950 for (RegionAllocator& allocator : m_allocators) {
951 using FunctionResultType = decltype(function(allocator));
952 if constexpr (std::is_same<IterationStatus, FunctionResultType>::value) {
953 if (function(allocator) == IterationStatus::Done)
954 break;
955 } else {
956 static_assert(std::is_same<void, FunctionResultType>::value);
957 function(allocator);
958 }
959 }
960#else
961 function(m_allocator);
962#endif // ENABLE(JUMP_ISLANDS)
963 }
964
965#if ENABLE(JUMP_ISLANDS)
966 class Islands : public RedBlackTree<Islands, void*>::Node {
967 WTF_MAKE_FAST_ALLOCATED;
968 public:
969 void* key() { return jumpSourceLocation.dataLocation(); }
970 CodeLocationLabel<ExecutableMemoryPtrTag> jumpSourceLocation;
971 Vector<CodeLocationLabel<ExecutableMemoryPtrTag>> jumpIslands;
972 };
973#endif // ENABLE(JUMP_ISLANDS)
974
975 Lock m_lock;
976 PageReservation m_reservation;
977#if ENABLE(JUMP_ISLANDS)
978 size_t m_regionSize;
979 FixedVector<RegionAllocator> m_allocators;
980 RedBlackTree<Islands, void*> m_islandsForJumpSourceLocation;
981#else
982 Allocator m_allocator;
983#endif // ENABLE(JUMP_ISLANDS)
984 size_t m_bytesReserved { 0 };
985#if USE(LIBPAS_JIT_HEAP)
986 std::atomic<size_t> m_bytesAllocated { 0 };
987#endif
988};
989
990// Keep this pointer in a mutable global variable to help Leaks find it.
991// But we do not use this pointer.
992static FixedVMPoolExecutableAllocator* globalFixedVMPoolExecutableAllocatorToWorkAroundLeaks = nullptr;
993void ExecutableAllocator::initializeUnderlyingAllocator()
994{
995 RELEASE_ASSERT(!g_jscConfig.fixedVMPoolExecutableAllocator);
996 g_jscConfig.fixedVMPoolExecutableAllocator = new FixedVMPoolExecutableAllocator();
997 globalFixedVMPoolExecutableAllocatorToWorkAroundLeaks = g_jscConfig.fixedVMPoolExecutableAllocator;
998}
999
1000bool ExecutableAllocator::isValid() const
1001{
1002 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1003 if (!allocator)
1004 return Base::isValid();
1005 return allocator->isValid();
1006}
1007
1008bool ExecutableAllocator::underMemoryPressure()
1009{
1010 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1011 if (!allocator)
1012 return Base::underMemoryPressure();
1013 return allocator->bytesAllocated() > allocator->bytesReserved() / 2;
1014}
1015
1016double ExecutableAllocator::memoryPressureMultiplier(size_t addedMemoryUsage)
1017{
1018 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1019 if (!allocator)
1020 return Base::memoryPressureMultiplier(addedMemoryUsage);
1021 ASSERT(allocator->bytesAllocated() <= allocator->bytesReserved());
1022 size_t bytesAllocated = allocator->bytesAllocated() + addedMemoryUsage;
1023 size_t bytesAvailable = allocator->bytesAvailable();
1024 if (bytesAllocated >= bytesAvailable)
1025 bytesAllocated = bytesAvailable;
1026 double result = 1.0;
1027 size_t divisor = bytesAvailable - bytesAllocated;
1028 if (divisor)
1029 result = static_cast<double>(bytesAvailable) / divisor;
1030 if (result < 1.0)
1031 result = 1.0;
1032 return result;
1033}
1034
1035RefPtr<ExecutableMemoryHandle> ExecutableAllocator::allocate(size_t sizeInBytes, JITCompilationEffort effort)
1036{
1037 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1038 if (!allocator)
1039 return Base::allocate(sizeInBytes, effort);
1040#if !USE(LIBPAS_JIT_HEAP)
1041 if (Options::logExecutableAllocation()) {
1042 MetaAllocator::Statistics stats = allocator->currentStatistics();
1043 dataLog("Allocating ", sizeInBytes, " bytes of executable memory with ", stats.bytesAllocated, " bytes allocated, ", stats.bytesReserved, " bytes reserved, and ", stats.bytesCommitted, " committed.\n");
1044 }
1045#endif
1046
1047 if (effort != JITCompilationCanFail && Options::reportMustSucceedExecutableAllocations()) {
1048 dataLog("Allocating ", sizeInBytes, " bytes of executable memory with JITCompilationMustSucceed.\n");
1049 WTFReportBacktrace();
1050 }
1051
1052 if (effort == JITCompilationCanFail
1053 && doExecutableAllocationFuzzingIfEnabled() == PretendToFailExecutableAllocation)
1054 return nullptr;
1055
1056 if (effort == JITCompilationCanFail) {
1057 // Don't allow allocations if we are down to reserve.
1058 size_t bytesAllocated = allocator->bytesAllocated() + sizeInBytes;
1059 size_t bytesAvailable = allocator->bytesAvailable();
1060 if (bytesAllocated > bytesAvailable) {
1061 if (Options::logExecutableAllocation())
1062 dataLog("Allocation failed because bytes allocated ", bytesAllocated, " > ", bytesAvailable, " bytes available.\n");
1063 return nullptr;
1064 }
1065 }
1066
1067 RefPtr<ExecutableMemoryHandle> result = allocator->allocate(sizeInBytes);
1068 if (!result) {
1069 if (effort != JITCompilationCanFail) {
1070 dataLog("Ran out of executable memory while allocating ", sizeInBytes, " bytes.\n");
1071 CRASH();
1072 }
1073 return nullptr;
1074 }
1075
1076 void* start = allocator->memoryStart();
1077 void* end = allocator->memoryEnd();
1078 void* resultStart = result->start().untaggedPtr();
1079 void* resultEnd = result->end().untaggedPtr();
1080 RELEASE_ASSERT(start <= resultStart && resultStart < end);
1081 RELEASE_ASSERT(start < resultEnd && resultEnd <= end);
1082 return result;
1083}
1084
1085bool ExecutableAllocator::isValidExecutableMemory(const AbstractLocker& locker, void* address)
1086{
1087 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1088 if (!allocator)
1089 return Base::isValidExecutableMemory(locker, address);
1090 return allocator->isInAllocatedMemory(locker, address);
1091}
1092
1093Lock& ExecutableAllocator::getLock() const
1094{
1095 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1096 if (!allocator)
1097 return Base::getLock();
1098 return allocator->getLock();
1099}
1100
1101size_t ExecutableAllocator::committedByteCount()
1102{
1103#if USE(LIBPAS_JIT_HEAP)
1104 return Base::committedByteCount();
1105#else // USE(LIBPAS_JIT_HEAP) -> so start of !USE(LIBPAS_JIT_HEAP)
1106 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1107 if (!allocator)
1108 return Base::committedByteCount();
1109 return allocator->bytesCommitted();
1110#endif // USE(LIBPAS_JIT_HEAP) -> so end of !USE(LIBPAS_JIT_HEAP)
1111}
1112
1113#if ENABLE(META_ALLOCATOR_PROFILE)
1114void ExecutableAllocator::dumpProfile()
1115{
1116 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1117 if (!allocator)
1118 return;
1119 allocator->dumpProfile();
1120}
1121#endif
1122
1123#if ENABLE(JUMP_ISLANDS)
1124void* ExecutableAllocator::getJumpIslandTo(void* from, void* newDestination)
1125{
1126 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1127 if (!allocator)
1128 RELEASE_ASSERT_NOT_REACHED();
1129
1130 return allocator->makeIsland(bitwise_cast<uintptr_t>(from), bitwise_cast<uintptr_t>(newDestination), false);
1131}
1132
1133void* ExecutableAllocator::getJumpIslandToConcurrently(void* from, void* newDestination)
1134{
1135 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1136 if (!allocator)
1137 RELEASE_ASSERT_NOT_REACHED();
1138
1139 return allocator->makeIsland(bitwise_cast<uintptr_t>(from), bitwise_cast<uintptr_t>(newDestination), true);
1140}
1141#endif
1142
1143void* startOfFixedExecutableMemoryPoolImpl()
1144{
1145 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1146 if (!allocator)
1147 return nullptr;
1148 return allocator->memoryStart();
1149}
1150
1151void* endOfFixedExecutableMemoryPoolImpl()
1152{
1153 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1154 if (!allocator)
1155 return nullptr;
1156 return allocator->memoryEnd();
1157}
1158
1159void dumpJITMemory(const void* dst, const void* src, size_t size)
1160{
1161 RELEASE_ASSERT(Options::dumpJITMemoryPath());
1162
1163#if OS(DARWIN)
1164 static Lock dumpJITMemoryLock;
1165 static int fd WTF_GUARDED_BY_LOCK(dumpJITMemoryLock) = -1;
1166 static uint8_t* buffer;
1167 static constexpr size_t bufferSize = fixedExecutableMemoryPoolSize;
1168 static size_t offset WTF_GUARDED_BY_LOCK(dumpJITMemoryLock) = 0;
1169 static bool needsToFlush WTF_GUARDED_BY_LOCK(dumpJITMemoryLock) = false;
1170 static LazyNeverDestroyed<Ref<WorkQueue>> flushQueue;
1171 struct DumpJIT {
1172 static void flush() WTF_REQUIRES_LOCK(dumpJITMemoryLock)
1173 {
1174 if (fd == -1) {
1175 auto path = String::fromLatin1(Options::dumpJITMemoryPath());
1176 path = makeStringByReplacingAll(path, "%pid"_s, String::number(getCurrentProcessID()));
1177 fd = open(FileSystem::fileSystemRepresentation(path).data(), O_CREAT | O_TRUNC | O_APPEND | O_WRONLY | O_EXLOCK | O_NONBLOCK, 0666);
1178 RELEASE_ASSERT(fd != -1);
1179 }
1180 ::write(fd, buffer, offset);
1181 offset = 0;
1182 needsToFlush = false;
1183 }
1184
1185 static void enqueueFlush() WTF_REQUIRES_LOCK(dumpJITMemoryLock)
1186 {
1187 if (needsToFlush)
1188 return;
1189
1190 needsToFlush = true;
1191 flushQueue.get()->dispatchAfter(Seconds(Options::dumpJITMemoryFlushInterval()), [] {
1192 Locker locker { dumpJITMemoryLock };
1193 if (!needsToFlush)
1194 return;
1195 flush();
1196 });
1197 }
1198
1199 static void write(const void* src, size_t size) WTF_REQUIRES_LOCK(dumpJITMemoryLock)
1200 {
1201 if (UNLIKELY(offset + size > bufferSize))
1202 flush();
1203 memcpy(buffer + offset, src, size);
1204 offset += size;
1205 enqueueFlush();
1206 }
1207 };
1208
1209 static std::once_flag once;
1210 std::call_once(once, [] {
1211 buffer = bitwise_cast<uint8_t*>(malloc(bufferSize));
1212 flushQueue.construct(WorkQueue::create("jsc.dumpJITMemory.queue", WorkQueue::QOS::Background));
1213 std::atexit([] {
1214 Locker locker { dumpJITMemoryLock };
1215 DumpJIT::flush();
1216 close(fd);
1217 fd = -1;
1218 });
1219 });
1220
1221 Locker locker { dumpJITMemoryLock };
1222 uint64_t time = mach_absolute_time();
1223 uint64_t dst64 = bitwise_cast<uintptr_t>(dst);
1224 uint64_t size64 = size;
1225 TraceScope(DumpJITMemoryStart, DumpJITMemoryStop, time, dst64, size64);
1226 DumpJIT::write(&time, sizeof(time));
1227 DumpJIT::write(&dst64, sizeof(dst64));
1228 DumpJIT::write(&size64, sizeof(size64));
1229 DumpJIT::write(src, size);
1230#else
1231 UNUSED_PARAM(dst);
1232 UNUSED_PARAM(src);
1233 UNUSED_PARAM(size);
1234 RELEASE_ASSERT_NOT_REACHED();
1235#endif
1236}
1237
1238#if USE(LIBPAS_JIT_HEAP)
1239RefPtr<ExecutableMemoryHandle> ExecutableMemoryHandle::createImpl(size_t sizeInBytes)
1240{
1241 void* key = jit_heap_try_allocate(sizeInBytes);
1242 if (!key)
1243 return nullptr;
1244 return adoptRef(new ExecutableMemoryHandle(MemoryPtr::makeFromRawPointer(key), jit_heap_get_size(key)));
1245}
1246
1247ExecutableMemoryHandle::~ExecutableMemoryHandle()
1248{
1249 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1250 allocator->handleWillBeReleased(*this, sizeInBytes());
1251 jit_heap_deallocate(key());
1252}
1253
1254void ExecutableMemoryHandle::shrink(size_t newSizeInBytes)
1255{
1256 size_t oldSizeInBytes = sizeInBytes();
1257 jit_heap_shrink(key(), newSizeInBytes);
1258 m_sizeInBytes = jit_heap_get_size(key());
1259 if (oldSizeInBytes != sizeInBytes()) {
1260 FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
1261 allocator->shrinkBytesAllocated(oldSizeInBytes, sizeInBytes());
1262 }
1263}
1264#endif
1265
1266} // namespace JSC
1267
1268#endif // ENABLE(JIT)
1269
1270namespace JSC {
1271
1272// Keep this pointer in a mutable global variable to help Leaks find it.
1273// But we do not use this pointer.
1274static ExecutableAllocator* globalExecutableAllocatorToWorkAroundLeaks = nullptr;
1275void ExecutableAllocator::initialize()
1276{
1277 g_jscConfig.executableAllocator = new ExecutableAllocator;
1278 globalExecutableAllocatorToWorkAroundLeaks = g_jscConfig.executableAllocator;
1279}
1280
1281ExecutableAllocator& ExecutableAllocator::singleton()
1282{
1283 ASSERT(g_jscConfig.executableAllocator);
1284 return *g_jscConfig.executableAllocator;
1285}
1286
1287} // namespace JSC
Note: See TracBrowser for help on using the repository browser.