Ignore:
Timestamp:
Apr 7, 2022, 10:03:19 AM (3 years ago)
Author:
[email protected]
Message:

[JSC][ARMv7] Support proper near calls and JUMP_ISLANDS
​https://bugs.webkit.org/show_bug.cgi?id=238143

Patch by Geza Lore <Geza Lore> on 2022-04-07
Reviewed by Yusuke Suzuki.

JSTests:

  • microbenchmarks/let-const-tdz-environment-parsing-and-hash-consing-speed.js:

Source/JavaScriptCore:

Implement nearCall and nearTailCall as single instruction direct
branches on ARMv7/Thumb-2. (Will need to support these for Wasm JITs,
to implement threadSafePatchableNearcall.) To make this possible while
also having an executable pool size larger than the branch range, I
also ported JUMP_ISLANDS.

To port JUMP_ISLANDS, a reformulation of the region allocations was
necessary, which is now done in terms of the range of the
nearCall/nearTailCall macroassembler macros. For ARM64, the behaviour
should be identical.

The jump islad reservation on ARMv7 is set to 5% of executable memory
size, which is approximately the same as the baseline JIT code size
saving provided by using short branches for near calls, so the change
should be neutral overall with respect to executable memory
consumption.

Also made it possible for the --jitMemoryReservationSize option to
request JIT memory that is larger than the default hardcoded size
while using JUMP_ISLANDS (we need this for testing on ARMv7, which has
a smaller default executable pool size). To do this the region
allocators are no longer statically allocated but are held in a
FixedVector.

Also removed the unused repatchCompact methods from assemblers.

  • assembler/ARM64Assembler.h:
  • assembler/ARMv7Assembler.h:

(JSC::ARMv7Assembler::isEven):
(JSC::ARMv7Assembler::makeEven):
(JSC::ARMv7Assembler::bl):
(JSC::ARMv7Assembler::link):
(JSC::ARMv7Assembler::linkTailCall):
(JSC::ARMv7Assembler::linkCall):
(JSC::ARMv7Assembler::relinkCall):
(JSC::ARMv7Assembler::relinkTailCall):
(JSC::ARMv7Assembler::prepareForAtomicRelinkJumpConcurrently):
(JSC::ARMv7Assembler::prepareForAtomicRelinkCallConcurrently):
(JSC::ARMv7Assembler::replaceWithJump):
(JSC::ARMv7Assembler::canEmitJump):
(JSC::ARMv7Assembler::isBL):
(JSC::ARMv7Assembler::linkJumpT4):
(JSC::ARMv7Assembler::linkConditionalJumpT4):
(JSC::ARMv7Assembler::linkJumpAbsolute):
(JSC::ARMv7Assembler::linkBranch):

  • assembler/AbstractMacroAssembler.h:

(JSC::AbstractMacroAssembler::repatchNearCall):

  • assembler/AssemblerCommon.h:

(JSC::isInt):

  • assembler/MIPSAssembler.h:
  • assembler/MacroAssemblerARM64.h:
  • assembler/MacroAssemblerARMv7.h:

(JSC::MacroAssemblerARMv7::nearCall):
(JSC::MacroAssemblerARMv7::nearTailCall):
(JSC::MacroAssemblerARMv7::linkCall):

  • assembler/MacroAssemblerMIPS.h:
  • assembler/MacroAssemblerRISCV64.h:
  • assembler/MacroAssemblerX86Common.h:
  • assembler/X86Assembler.h:
  • bytecode/Repatch.cpp:

(JSC::linkPolymorphicCall):

  • jit/ExecutableAllocator.cpp:

(JSC::initializeJITPageReservation):

Source/WTF:

Support constructor arguments for FixedVector element initialization.

  • wtf/EmbeddedFixedVector.h:
  • wtf/FixedVector.h:

(WTF::FixedVector::FixedVector):

  • wtf/PlatformEnable.h:
  • wtf/TrailingArray.h:

(WTF::TrailingArray::TrailingArray):

  • wtf/Vector.h:

(WTF::VectorTypeOperations::initializeWithArgs):

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/JavaScriptCore/assembler/ARMv7Assembler.h

    r291946 r292540  
    491491
    492492private:
     493
     494    // In the ARMv7 ISA, the LSB of a code pointer indicates whether the target uses Thumb vs ARM
     495    // encoding. These utility functions are there for when we need to deal with this.
     496    static bool isEven(const void* ptr) { return !(reinterpret_cast<uintptr_t>(ptr) & 1); }
     497    static bool isEven(AssemblerLabel &label) { return !(label.offset() & 1); }
     498    static void* makeEven(const void* ptr)
     499    {
     500        ASSERT(!isEven(ptr));
     501        return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(ptr) & ~1);
     502    }
    493503
    494504    // ARMv7, Appx-A.6.3
    … …  
    610620        OP_B_T3a        = 0xF000,
    611621        OP_B_T4a        = 0xF000,
     622        OP_BL_T4a       = 0xF000,
    612623        OP_AND_imm_T1   = 0xF000,
    613624        OP_TST_imm      = 0xF010,
    … …  
    698709        OP_B_T3b         = 0x8000,
    699710        OP_B_T4b         = 0x9000,
     711        OP_BL_T4b        = 0xD000,
    700712    } OpcodeID2;
    701713
    … …  
    718730            };
    719731        } m_u;
     732    };
     733
     734    enum class BranchWithLink : bool {
     735        No = false,
     736        Yes = true
    720737    };
    721738
    … …  
    920937    {
    921938        m_formatter.twoWordOp16Op16(OP_B_T4a, OP_B_T4b);
     939        return m_formatter.label();
     940    }
     941
     942    // Only allowed in IT (if then) block if last instruction.
     943    ALWAYS_INLINE AssemblerLabel bl()
     944    {
     945        m_formatter.twoWordOp16Op16(OP_BL_T4a, OP_BL_T4b);
    922946        return m_formatter.label();
    923947    }
    … …  
    22742298            break;
    22752299        case LinkJumpT4:
    2276             linkJumpT4<copy>(reinterpret_cast_ptr<uint16_t*>(from), fromInstruction, to);
     2300            linkJumpT4<copy>(reinterpret_cast_ptr<uint16_t*>(from), fromInstruction, to, BranchWithLink::No);
    22772301            break;
    22782302        case LinkConditionalJumpT4:
    … …  
    23222346    }
    23232347
     2348    static void linkTailCall(void* code, AssemblerLabel from, void* to)
     2349    {
     2350        ASSERT(from.isSet());
     2351
     2352        uint16_t* location = reinterpret_cast<uint16_t*>(reinterpret_cast<intptr_t>(code) + from.offset());
     2353        linkBranch(location, location, makeEven(to), BranchWithLink::No);
     2354    }
     2355
    23242356    static void linkCall(void* code, AssemblerLabel from, void* to)
    23252357    {
    2326         ASSERT(!(reinterpret_cast<intptr_t>(code) & 1));
    23272358        ASSERT(from.isSet());
    23282359
    2329         setPointer(reinterpret_cast<uint16_t*>(reinterpret_cast<intptr_t>(code) + from.offset()) - 1, to, false);
     2360        uint16_t* location = reinterpret_cast<uint16_t*>(reinterpret_cast<intptr_t>(code) + from.offset());
     2361        linkBranch(location, location, makeEven(to), BranchWithLink::Yes);
    23302362    }
    23312363
    … …  
    23512383    static void relinkCall(void* from, void* to)
    23522384    {
    2353         ASSERT(!(reinterpret_cast<intptr_t>(from) & 1));
    2354 
    2355         setPointer(reinterpret_cast<uint16_t*>(from) - 1, to, true);
    2356     }
    2357    
     2385        ASSERT(isEven(from));
     2386
     2387        uint16_t* location = reinterpret_cast<uint16_t*>(from);
     2388        if (isBL(location - 2)) {
     2389            linkBranch(location, location, makeEven(to), BranchWithLink::Yes);
     2390            cacheFlush(location - 2, 2 * sizeof(uint16_t));
     2391            return;
     2392        }
     2393
     2394        setPointer(location - 1, to, true);
     2395    }
     2396
     2397    static void relinkTailCall(void* from, void* to)
     2398    {
     2399        ASSERT(isEven(from));
     2400
     2401        uint16_t* location = reinterpret_cast<uint16_t*>(from);
     2402        linkBranch(location, location, to, BranchWithLink::No);
     2403        cacheFlush(location - 2, 2 * sizeof(uint16_t));
     2404    }
     2405
     2406#if ENABLE(JUMP_ISLANDS)
     2407    static void* prepareForAtomicRelinkJumpConcurrently(void* from, void* to)
     2408    {
     2409        ASSERT(isEven(from));
     2410        ASSERT(isEven(to));
     2411
     2412        intptr_t offset = bitwise_cast<intptr_t>(to) - bitwise_cast<intptr_t>(from);
     2413        ASSERT(static_cast<int>(offset) == offset);
     2414
     2415        if (isInt<25>(offset))
     2416            return to;
     2417
     2418        return ExecutableAllocator::singleton().getJumpIslandToConcurrently(from, to);
     2419    }
     2420
     2421    static void* prepareForAtomicRelinkCallConcurrently(void* from, void* to)
     2422    {
     2423        ASSERT(isEven(from));
     2424
     2425        return prepareForAtomicRelinkJumpConcurrently(from, makeEven(to));
     2426    }
     2427#endif
     2428
    23582429    static void* readCallTarget(void* from)
    23592430    {
    … …  
    23682439    }
    23692440   
    2370     static void repatchCompact(void* where, int32_t offset)
    2371     {
    2372         ASSERT(offset >= -255 && offset <= 255);
    2373 
    2374         bool add = true;
    2375         if (offset < 0) {
    2376             add = false;
    2377             offset = -offset;
    2378         }
    2379        
    2380         offset |= (add << 9);
    2381         offset |= (1 << 10);
    2382         offset |= (1 << 11);
    2383 
    2384         uint16_t* location = reinterpret_cast<uint16_t*>(where);
    2385         uint16_t instruction = location[1] & ~((1 << 12) - 1);
    2386         instruction |= offset;
    2387         performJITMemcpy(location + 1, &instruction, sizeof(uint16_t));
    2388         cacheFlush(location, sizeof(uint16_t) * 2);
    2389     }
    2390 
    23912441    static void repatchPointer(void* where, void* value)
    23922442    {
    … …  
    24092459        if (canBeJumpT4(reinterpret_cast<uint16_t*>(instructionStart), to)) {
    24102460            uint16_t* ptr = reinterpret_cast<uint16_t*>(instructionStart) + 2;
    2411             linkJumpT4(ptr, ptr, to);
     2461            linkJumpT4(ptr, ptr, to, BranchWithLink::No);
    24122462            cacheFlush(ptr - 2, sizeof(uint16_t) * 2);
    24132463        } else {
    … …  
    24182468#else
    24192469        uint16_t* ptr = reinterpret_cast<uint16_t*>(instructionStart) + 2;
    2420         linkJumpT4(ptr, ptr, to);
     2470        linkJumpT4(ptr, ptr, to, BranchWithLink::No);
    24212471        cacheFlush(ptr - 2, sizeof(uint16_t) * 2);
    24222472#endif
    … …  
    25292579    }
    25302580
     2581    static ALWAYS_INLINE bool canEmitJump(void* from, void* to)
     2582    {
     2583        // 'from' holds the address of the branch instruction. The branch range however is relative
     2584        // to the architectural value of the PC which is 4 larger than the address of the branch.
     2585        intptr_t offset = bitwise_cast<intptr_t>(to) - (bitwise_cast<intptr_t>(from) + 4);
     2586        return isInt<25>(offset);
     2587    }
     2588
    25312589private:
    25322590    // VFP operations commonly take one or more 5-bit operands, typically representing a
    2533     // floating point register number.  This will commonly be encoded in the instruction
    2534     // in two parts, with one single bit field, and one 4-bit field.  In the case of
     2591    // floating point register number. This will commonly be encoded in the instruction
     2592    // in two parts, with one single bit field, and one 4-bit field. In the case of
    25352593    // double precision operands the high bit of the register number will be encoded
    25362594    // separately, and for single precision operands the high bit of the register number
    … …  
    26552713    }
    26562714
     2715    static bool isBL(const void* address)
     2716    {
     2717        const uint16_t* instruction = static_cast<const uint16_t*>(address);
     2718        return ((instruction[0] & 0xf800) == OP_BL_T4a) && ((instruction[1] & 0xd000) == OP_BL_T4b);
     2719    }
     2720
    26572721    static bool isBX(const void* address)
    26582722    {
    … …  
    27882852   
    27892853    template<CopyFunction copy = performJITMemcpy>
    2790     static void linkJumpT4(uint16_t* writeTarget, const uint16_t* instruction, void* target)
     2854    static void linkJumpT4(uint16_t* writeTarget, const uint16_t* instruction, void* target, BranchWithLink link)
    27912855    {
    27922856        // FIMXE: this should be up in the MacroAssembler layer. :-(       
    … …  
    28042868        uint16_t instructions[2];
    28052869        instructions[0] = OP_B_T4a | ((relative & 0x1000000) >> 14) | ((relative & 0x3ff000) >> 12);
    2806         instructions[1] = OP_B_T4b | ((relative & 0x800000) >> 10) | ((relative & 0x400000) >> 11) | ((relative & 0xffe) >> 1);
     2870        instructions[1] = OP_B_T4b | (static_cast<uint16_t>(link) << 14) | ((relative & 0x800000) >> 10) | ((relative & 0x400000) >> 11) | ((relative & 0xffe) >> 1);
    28072871        copy(writeTarget - 2, instructions, 2 * sizeof(uint16_t));
    28082872    }
    … …  
    28172881        uint16_t newInstruction = ifThenElse(cond) | OP_IT;
    28182882        copy(writeTarget - 3, &newInstruction, sizeof(uint16_t));
    2819         linkJumpT4<copy>(writeTarget, instruction, target);
     2883        linkJumpT4<copy>(writeTarget, instruction, target, BranchWithLink::No);
    28202884    }
    28212885
    … …  
    28732937            instructions[2] = OP_NOP_T2b;
    28742938            performJITMemcpy(writeTarget - 5, instructions, 3 * sizeof(uint16_t));
    2875             linkJumpT4(writeTarget, instruction, target);
     2939            linkJumpT4(writeTarget, instruction, target, BranchWithLink::No);
    28762940        } else {
    28772941            const uint16_t JUMP_TEMPORARY_REGISTER = ARMRegisters::ip;
    … …  
    28882952        }
    28892953    }
    2890    
     2954
     2955    static void linkBranch(uint16_t* from, const uint16_t* fromInstruction, void* to, BranchWithLink link)
     2956    {
     2957        ASSERT(isEven(fromInstruction));
     2958        ASSERT(isEven(from));
     2959        ASSERT(isEven(to));
     2960        ASSERT(link == BranchWithLink::Yes ? isBL(from - 2) : isB(from - 2));
     2961
     2962        intptr_t offset = bitwise_cast<intptr_t>(to) - bitwise_cast<intptr_t>(fromInstruction);
     2963#if ENABLE(JUMP_ISLANDS)
     2964        if (!isInt<25>(offset)) {
     2965            to = ExecutableAllocator::singleton().getJumpIslandTo(bitwise_cast<void*>(fromInstruction), to);
     2966            offset = bitwise_cast<intptr_t>(to) - bitwise_cast<intptr_t>(fromInstruction);
     2967        }
     2968#endif
     2969        RELEASE_ASSERT(isInt<25>(offset));
     2970
     2971        linkJumpT4(from, fromInstruction, to, link);
     2972    }
     2973
    28912974    static uint16_t twoWordOp5i6Imm4Reg4EncodedImmFirst(uint16_t op, ARMThumbImmediate imm)
    28922975    {
Note: See TracChangeset for help on using the changeset viewer.