1/*
2 * Copyright (C) 2015-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "CallFrameShuffler.h"
28
29#if ENABLE(JIT)
30
31#include "CachedRecovery.h"
32#include "CCallHelpers.h"
33#include "CodeBlock.h"
34
35namespace JSC {
36
37CallFrameShuffler::CallFrameShuffler(CCallHelpers& jit, const CallFrameShuffleData& data)
38 : m_jit(jit)
39 , m_oldFrame(data.numLocals + CallerFrameAndPC::sizeInRegisters, nullptr)
40 , m_newFrame(data.args.size() + CallFrame::headerSizeInRegisters, nullptr)
41 , m_alignedOldFrameSize(CallFrame::headerSizeInRegisters
42 + roundArgumentCountToAlignFrame(jit.codeBlock()->numParameters()))
43 , m_alignedNewFrameSize(CallFrame::headerSizeInRegisters
44 + roundArgumentCountToAlignFrame(data.args.size()))
45 , m_frameDelta(m_alignedNewFrameSize - m_alignedOldFrameSize)
46 , m_lockedRegisters(RegisterSet::allRegisters())
47 , m_numPassedArgs(data.numPassedArgs)
48{
49 // We are allowed all the usual registers...
50 for (unsigned i = GPRInfo::numberOfRegisters; i--; )
51 m_lockedRegisters.clear(GPRInfo::toRegister(i));
52 for (unsigned i = FPRInfo::numberOfRegisters; i--; )
53 m_lockedRegisters.clear(FPRInfo::toRegister(i));
54
55#if USE(JSVALUE64)
56 // ... as well as the runtime registers on 64-bit architectures.
57 // However do not use these registers on 32-bit architectures since
58 // saving and restoring callee-saved registers in CallFrameShuffler isn't supported
59 // on 32-bit architectures yet.
60 m_lockedRegisters.exclude(RegisterSet::vmCalleeSaveRegisters());
61#endif
62
63 ASSERT(!data.callee.isInJSStack() || data.callee.virtualRegister().isLocal());
64 addNew(VirtualRegister(CallFrameSlot::callee), data.callee);
65
66 for (size_t i = 0; i < data.args.size(); ++i) {
67 ASSERT(!data.args[i].isInJSStack() || data.args[i].virtualRegister().isLocal());
68 addNew(virtualRegisterForArgument(i), data.args[i]);
69 }
70
71#if USE(JSVALUE64)
72 for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
73 if (!data.registers[reg].isSet())
74 continue;
75
76 if (reg.isGPR())
77 addNew(JSValueRegs(reg.gpr()), data.registers[reg]);
78 else
79 addNew(reg.fpr(), data.registers[reg]);
80 }
81
82 m_tagTypeNumber = data.tagTypeNumber;
83 if (m_tagTypeNumber != InvalidGPRReg)
84 lockGPR(m_tagTypeNumber);
85#endif
86}
87
88void CallFrameShuffler::dump(PrintStream& out) const
89{
90 static const char* delimiter = " +-------------------------------+ ";
91 static const char* dangerDelimiter = " X-------------------------------X ";
92 static const char* dangerBoundsDelimiter = " XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX ";
93 static const char* emptySpace = " ";
94 out.print(" ");
95 out.print(" Old frame ");
96 out.print(" New frame ");
97 out.print("\n");
98 int totalSize = m_alignedOldFrameSize + std::max(numLocals(), m_alignedNewFrameSize) + 3;
99 for (int i = 0; i < totalSize; ++i) {
100 VirtualRegister old { m_alignedOldFrameSize - i - 1 };
101 VirtualRegister newReg { old + m_frameDelta };
102
103 if (!isValidOld(old) && old != firstOld() - 1
104 && !isValidNew(newReg) && newReg != firstNew() - 1)
105 continue;
106
107 out.print(" ");
108 if (dangerFrontier() >= firstNew()
109 && (newReg == dangerFrontier() || newReg == firstNew() - 1))
110 out.print(dangerBoundsDelimiter);
111 else if (isValidOld(old))
112 out.print(isValidNew(newReg) && isDangerNew(newReg) ? dangerDelimiter : delimiter);
113 else if (old == firstOld() - 1)
114 out.print(delimiter);
115 else
116 out.print(emptySpace);
117 if (dangerFrontier() >= firstNew()
118 && (newReg == dangerFrontier() || newReg == firstNew() - 1))
119 out.print(dangerBoundsDelimiter);
120 else if (isValidNew(newReg) || newReg == firstNew() - 1)
121 out.print(isDangerNew(newReg) ? dangerDelimiter : delimiter);
122 else
123 out.print(emptySpace);
124 out.print("\n");
125 if (old == firstOld())
126 out.print(" sp --> ");
127 else if (!old.offset())
128 out.print(" fp --> ");
129 else
130 out.print(" ");
131 if (isValidOld(old)) {
132 if (getOld(old)) {
133 auto str = toCString(old);
134 if (isValidNew(newReg) && isDangerNew(newReg))
135 out.printf(" X %18s X ", str.data());
136 else
137 out.printf(" | %18s | ", str.data());
138 } else if (isValidNew(newReg) && isDangerNew(newReg))
139 out.printf(" X%30s X ", "");
140 else
141 out.printf(" |%30s | ", "");
142 } else
143 out.print(emptySpace);
144 if (isValidNew(newReg)) {
145 const char d = isDangerNew(newReg) ? 'X' : '|';
146 auto str = toCString(newReg);
147 if (getNew(newReg)) {
148 if (getNew(newReg)->recovery().isConstant())
149 out.printf(" %c%8s <- constant %c ", d, str.data(), d);
150 else {
151 auto recoveryStr = toCString(getNew(newReg)->recovery());
152 out.printf(" %c%8s <- %18s %c ", d, str.data(),
153 recoveryStr.data(), d);
154 }
155 } else if (newReg == VirtualRegister { CallFrameSlot::argumentCount })
156 out.printf(" %c%8s <- %18zu %c ", d, str.data(), argCount(), d);
157 else
158 out.printf(" %c%30s %c ", d, "", d);
159 } else
160 out.print(emptySpace);
161 if (newReg == firstNew() - m_newFrameOffset && !isSlowPath())
162 out.print(" <-- new sp before jump (current ", m_newFrameBase, ") ");
163 if (newReg == firstNew())
164 out.print(" <-- new fp after prologue");
165 out.print("\n");
166 }
167 out.print(" ");
168 out.print(" Live registers ");
169 out.print(" Wanted registers ");
170 out.print("\n");
171 for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
172 CachedRecovery* oldCachedRecovery { m_registers[reg] };
173 CachedRecovery* newCachedRecovery { m_newRegisters[reg] };
174 if (!oldCachedRecovery && !newCachedRecovery)
175 continue;
176 out.print(" ");
177 if (oldCachedRecovery) {
178 auto str = toCString(reg);
179 out.printf(" %8s ", str.data());
180 } else
181 out.print(emptySpace);
182#if USE(JSVALUE32_64)
183 if (newCachedRecovery) {
184 JSValueRegs wantedJSValueRegs { newCachedRecovery->wantedJSValueRegs() };
185 if (reg.isFPR())
186 out.print(reg, " <- ", newCachedRecovery->recovery());
187 else {
188 if (reg.gpr() == wantedJSValueRegs.tagGPR())
189 out.print(reg.gpr(), " <- tag(", newCachedRecovery->recovery(), ")");
190 else
191 out.print(reg.gpr(), " <- payload(", newCachedRecovery->recovery(), ")");
192 }
193 }
194#else
195 if (newCachedRecovery)
196 out.print(" ", reg, " <- ", newCachedRecovery->recovery());
197#endif
198 out.print("\n");
199 }
200 out.print(" Locked registers: ");
201 bool firstLocked { true };
202 for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
203 if (m_lockedRegisters.get(reg)) {
204 out.print(firstLocked ? "" : ", ", reg);
205 firstLocked = false;
206 }
207 }
208 out.print("\n");
209
210 if (isSlowPath())
211 out.print(" Using fp-relative addressing for slow path call\n");
212 else
213 out.print(" Using sp-relative addressing for jump (using ", m_newFrameBase, " as new sp)\n");
214 if (m_oldFrameOffset)
215 out.print(" Old frame offset is ", m_oldFrameOffset, "\n");
216 if (m_newFrameOffset)
217 out.print(" New frame offset is ", m_newFrameOffset, "\n");
218#if USE(JSVALUE64)
219 if (m_tagTypeNumber != InvalidGPRReg)
220 out.print(" TagTypeNumber is currently in ", m_tagTypeNumber, "\n");
221#endif
222}
223
224CachedRecovery* CallFrameShuffler::getCachedRecovery(ValueRecovery recovery)
225{
226 ASSERT(!recovery.isConstant());
227 if (recovery.isInGPR())
228 return m_registers[recovery.gpr()];
229 if (recovery.isInFPR())
230 return m_registers[recovery.fpr()];
231#if USE(JSVALUE32_64)
232 if (recovery.technique() == InPair) {
233 ASSERT(m_registers[recovery.tagGPR()] == m_registers[recovery.payloadGPR()]);
234 return m_registers[recovery.payloadGPR()];
235 }
236#endif
237 ASSERT(recovery.isInJSStack());
238 return getOld(recovery.virtualRegister());
239}
240
241CachedRecovery* CallFrameShuffler::setCachedRecovery(ValueRecovery recovery, CachedRecovery* cachedRecovery)
242{
243 ASSERT(!recovery.isConstant());
244 if (recovery.isInGPR())
245 return m_registers[recovery.gpr()] = cachedRecovery;
246 if (recovery.isInFPR())
247 return m_registers[recovery.fpr()] = cachedRecovery;
248#if USE(JSVALUE32_64)
249 if (recovery.technique() == InPair) {
250 m_registers[recovery.tagGPR()] = cachedRecovery;
251 return m_registers[recovery.payloadGPR()] = cachedRecovery;
252 }
253#endif
254 ASSERT(recovery.isInJSStack());
255 setOld(recovery.virtualRegister(), cachedRecovery);
256 return cachedRecovery;
257}
258
259void CallFrameShuffler::spill(CachedRecovery& cachedRecovery)
260{
261 ASSERT(!isSlowPath());
262 ASSERT(cachedRecovery.recovery().isInRegisters());
263
264 VirtualRegister spillSlot { 0 };
265 for (VirtualRegister slot = firstOld(); slot <= lastOld(); slot += 1) {
266 if (slot >= newAsOld(firstNew()))
267 break;
268
269 if (getOld(slot))
270 continue;
271
272 spillSlot = slot;
273 break;
274 }
275 // We must have enough slots to be able to fit the whole callee's
276 // frame for the slow path - unless we are in the FTL. In that
277 // case, we are allowed to extend the frame *once*, since we are
278 // guaranteed to have enough available space for that.
279 if (spillSlot >= newAsOld(firstNew()) || !spillSlot.isLocal()) {
280 RELEASE_ASSERT(!m_didExtendFrame);
281 extendFrameIfNeeded();
282 spill(cachedRecovery);
283 return;
284 }
285
286 if (verbose)
287 dataLog(" * Spilling ", cachedRecovery.recovery(), " into ", spillSlot, "\n");
288 auto format = emitStore(cachedRecovery, addressForOld(spillSlot));
289 ASSERT(format != DataFormatNone);
290 updateRecovery(cachedRecovery, ValueRecovery::displacedInJSStack(spillSlot, format));
291}
292
293void CallFrameShuffler::emitDeltaCheck()
294{
295 if (ASSERT_DISABLED)
296 return;
297
298 GPRReg scratchGPR { getFreeGPR() };
299 if (scratchGPR != InvalidGPRReg) {
300 if (verbose)
301 dataLog(" Using ", scratchGPR, " for the fp-sp delta check\n");
302 m_jit.move(MacroAssembler::stackPointerRegister, scratchGPR);
303 m_jit.subPtr(GPRInfo::callFrameRegister, scratchGPR);
304 MacroAssembler::Jump ok = m_jit.branch32(
305 MacroAssembler::Equal, scratchGPR,
306 MacroAssembler::TrustedImm32(-numLocals() * sizeof(Register)));
307 m_jit.abortWithReason(JITUnexpectedCallFrameSize);
308 ok.link(&m_jit);
309 } else if (verbose)
310 dataLog(" Skipping the fp-sp delta check since there is too much pressure");
311}
312
313void CallFrameShuffler::extendFrameIfNeeded()
314{
315 ASSERT(!m_didExtendFrame);
316
317 VirtualRegister firstRead { firstOld() };
318 for (; firstRead <= virtualRegisterForLocal(0); firstRead += 1) {
319 if (getOld(firstRead))
320 break;
321 }
322 size_t availableSize = static_cast<size_t>(firstRead.offset() - firstOld().offset());
323 size_t wantedSize = m_newFrame.size() + m_newFrameOffset;
324
325 if (availableSize < wantedSize) {
326 size_t delta = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), wantedSize - availableSize);
327 m_oldFrame.grow(m_oldFrame.size() + delta);
328 for (size_t i = 0; i < delta; ++i)
329 m_oldFrame[m_oldFrame.size() - i - 1] = nullptr;
330 m_jit.subPtr(MacroAssembler::TrustedImm32(delta * sizeof(Register)), MacroAssembler::stackPointerRegister);
331
332 if (isSlowPath())
333 m_frameDelta = numLocals() + CallerFrameAndPC::sizeInRegisters;
334 else
335 m_oldFrameOffset = numLocals();
336
337 if (verbose)
338 dataLogF(" Not enough space - extending the old frame %zu slot\n", delta);
339 }
340
341 m_didExtendFrame = true;
342}
343
344void CallFrameShuffler::prepareForSlowPath()
345{
346 ASSERT(isUndecided());
347 emitDeltaCheck();
348
349 m_frameDelta = numLocals() + CallerFrameAndPC::sizeInRegisters;
350 m_newFrameBase = MacroAssembler::stackPointerRegister;
351 m_newFrameOffset = -CallerFrameAndPC::sizeInRegisters;
352
353 if (verbose)
354 dataLog("\n\nPreparing frame for slow path call:\n");
355
356 // When coming from the FTL, we need to extend the frame. In other
357 // cases, we may end up extending the frame if we previously
358 // spilled things (e.g. in polymorphic cache).
359 extendFrameIfNeeded();
360
361 if (verbose)
362 dataLog(*this);
363
364 prepareAny();
365
366 if (verbose)
367 dataLog("Ready for slow path call!\n");
368}
369
370void CallFrameShuffler::prepareForTailCall()
371{
372 ASSERT(isUndecided());
373 emitDeltaCheck();
374
375 // We'll use sp-based indexing so that we can load the
376 // caller's frame pointer into the fpr immediately
377 m_oldFrameBase = MacroAssembler::stackPointerRegister;
378 m_oldFrameOffset = numLocals();
379 m_newFrameBase = acquireGPR();
380#if CPU(X86)
381 // We load the frame pointer manually, but we need to ask the
382 // algorithm to move the return PC for us (it'd probably
383 // require a write to the danger zone). Since it'd be awkward
384 // to ask for half a value move, we ask that the whole thing
385 // be moved for us.
386 addNew(VirtualRegister { 0 },
387 ValueRecovery::displacedInJSStack(VirtualRegister(0), DataFormatJS));
388
389 // sp will point to head0 and we will move it up half a slot
390 // manually
391 m_newFrameOffset = 0;
392#elif CPU(ARM_THUMB2) || CPU(MIPS)
393 // We load the frame pointer and link register
394 // manually. We could ask the algorithm to load them for us,
395 // and it would allow us to use the link register as an extra
396 // temporary - but it'd mean that the frame pointer can also
397 // be used as an extra temporary, so we keep the link register
398 // locked instead.
399
400 // sp will point to head1 since the callee's prologue pushes
401 // the call frame and link register.
402 m_newFrameOffset = -1;
403#elif CPU(ARM64)
404 // We load the frame pointer and link register manually. We
405 // could ask the algorithm to load the link register for us
406 // (which would allow for its use as an extra temporary), but
407 // since its not in GPRInfo, we can't do it.
408
409 // sp will point to head2 since the callee's prologue pushes the
410 // call frame and link register
411 m_newFrameOffset = -2;
412#elif CPU(X86_64)
413 // We load the frame pointer manually, but we ask the
414 // algorithm to move the return PC for us (it'd probably
415 // require a write in the danger zone)
416 addNew(VirtualRegister { 1 },
417 ValueRecovery::displacedInJSStack(VirtualRegister(1), DataFormatJS));
418
419 // sp will point to head1 since the callee's prologue pushes
420 // the call frame register
421 m_newFrameOffset = -1;
422#else
423 UNREACHABLE_FOR_PLATFORM();
424#endif
425
426 if (verbose)
427 dataLog(" Emitting code for computing the new frame base\n");
428
429 // We compute the new frame base by first computing the top of the
430 // old frame (taking into account an argument count higher than
431 // the number of parameters), then substracting to it the aligned
432 // new frame size (adjusted).
433 m_jit.load32(MacroAssembler::Address(GPRInfo::callFrameRegister, CallFrameSlot::argumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset), m_newFrameBase);
434 MacroAssembler::Jump argumentCountOK =
435 m_jit.branch32(MacroAssembler::BelowOrEqual, m_newFrameBase,
436 MacroAssembler::TrustedImm32(m_jit.codeBlock()->numParameters()));
437 m_jit.add32(MacroAssembler::TrustedImm32(stackAlignmentRegisters() - 1 + CallFrame::headerSizeInRegisters), m_newFrameBase);
438 m_jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), m_newFrameBase);
439 m_jit.mul32(MacroAssembler::TrustedImm32(sizeof(Register)), m_newFrameBase, m_newFrameBase);
440 MacroAssembler::Jump done = m_jit.jump();
441 argumentCountOK.link(&m_jit);
442 m_jit.move(
443 MacroAssembler::TrustedImm32(m_alignedOldFrameSize * sizeof(Register)),
444 m_newFrameBase);
445 done.link(&m_jit);
446
447 m_jit.addPtr(GPRInfo::callFrameRegister, m_newFrameBase);
448 m_jit.subPtr(
449 MacroAssembler::TrustedImm32(
450 (m_alignedNewFrameSize + m_newFrameOffset) * sizeof(Register)),
451 m_newFrameBase);
452
453 // We load the link register manually for architectures that have one
454#if CPU(ARM_THUMB2) || CPU(ARM64)
455 m_jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()),
456 MacroAssembler::linkRegister);
457#if CPU(ARM64E)
458 m_jit.addPtr(MacroAssembler::TrustedImm32(sizeof(CallerFrameAndPC)), MacroAssembler::framePointerRegister);
459 m_jit.untagPtr(MacroAssembler::framePointerRegister, MacroAssembler::linkRegister);
460 m_jit.subPtr(MacroAssembler::TrustedImm32(sizeof(CallerFrameAndPC)), MacroAssembler::framePointerRegister);
461#endif
462
463#elif CPU(MIPS)
464 m_jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, sizeof(void*)),
465 MacroAssembler::returnAddressRegister);
466#endif
467
468 // We want the frame pointer to always point to a valid frame, and
469 // we are going to trash the current one. Let's make it point to
470 // our caller's frame, since that's what we want to end up with.
471 m_jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister),
472 MacroAssembler::framePointerRegister);
473
474 if (verbose)
475 dataLog("Preparing frame for tail call:\n", *this);
476
477 prepareAny();
478
479#if CPU(X86)
480 if (verbose)
481 dataLog(" Simulating pop of the call frame register\n");
482 m_jit.addPtr(MacroAssembler::TrustedImm32(sizeof(void*)), MacroAssembler::stackPointerRegister);
483#endif
484
485 if (verbose)
486 dataLog("Ready for tail call!\n");
487}
488
489bool CallFrameShuffler::tryWrites(CachedRecovery& cachedRecovery)
490{
491 ASSERT(m_newFrameBase != InvalidGPRReg);
492
493 // If the value is already set up correctly, we don't have
494 // anything to do.
495 if (isSlowPath() && cachedRecovery.recovery().isInJSStack()
496 && cachedRecovery.targets().size() == 1
497 && newAsOld(cachedRecovery.targets()[0]) == cachedRecovery.recovery().virtualRegister()) {
498 cachedRecovery.clearTargets();
499 if (!cachedRecovery.wantedJSValueRegs() && cachedRecovery.wantedFPR() == InvalidFPRReg)
500 clearCachedRecovery(cachedRecovery.recovery());
501 return true;
502 }
503
504 if (!canLoadAndBox(cachedRecovery))
505 return false;
506
507 emitLoad(cachedRecovery);
508 emitBox(cachedRecovery);
509 ASSERT(cachedRecovery.recovery().isInRegisters()
510 || cachedRecovery.recovery().isConstant());
511
512 if (verbose)
513 dataLog(" * Storing ", cachedRecovery.recovery());
514 for (size_t i = 0; i < cachedRecovery.targets().size(); ++i) {
515 VirtualRegister target { cachedRecovery.targets()[i] };
516 ASSERT(!isDangerNew(target));
517 if (verbose)
518 dataLog(!i ? " into " : ", and ", "NEW ", target);
519 emitStore(cachedRecovery, addressForNew(target));
520 setNew(target, nullptr);
521 }
522 if (verbose)
523 dataLog("\n");
524 cachedRecovery.clearTargets();
525 if (!cachedRecovery.wantedJSValueRegs() && cachedRecovery.wantedFPR() == InvalidFPRReg)
526 clearCachedRecovery(cachedRecovery.recovery());
527
528 return true;
529}
530
531bool CallFrameShuffler::performSafeWrites()
532{
533 VirtualRegister firstSafe;
534 VirtualRegister end { lastNew() + 1 };
535 Vector<VirtualRegister> failures;
536
537 // For all cachedRecoveries that writes to the safe zone, if it
538 // doesn't also write to the danger zone, we try to perform
539 // the writes. This may free up danger slots, so we iterate
540 // again until it doesn't happen anymore.
541 //
542 // Note that even though we have a while block, we look at
543 // each slot of the new call frame at most once since in each
544 // iteration beyond the first, we only load up the portion of
545 // the new call frame that was dangerous and became safe due
546 // to the previous iteration.
547 do {
548 firstSafe = dangerFrontier() + 1;
549 if (verbose)
550 dataLog(" Trying safe writes (between NEW ", firstSafe, " and NEW ", end - 1, ")\n");
551 bool didProgress = false;
552 for (VirtualRegister reg = firstSafe; reg < end; reg += 1) {
553 CachedRecovery* cachedRecovery = getNew(reg);
554 if (!cachedRecovery) {
555 if (verbose)
556 dataLog(" + ", reg, " is OK.\n");
557 continue;
558 }
559 if (!hasOnlySafeWrites(*cachedRecovery)) {
560 if (verbose) {
561 dataLog(" - ", cachedRecovery->recovery(), " writes to NEW ", reg,
562 " but also has dangerous writes.\n");
563 }
564 continue;
565 }
566 if (cachedRecovery->wantedJSValueRegs()) {
567 if (verbose) {
568 dataLog(" - ", cachedRecovery->recovery(), " writes to NEW ", reg,
569 " but is also needed in registers.\n");
570 }
571 continue;
572 }
573 if (cachedRecovery->wantedFPR() != InvalidFPRReg) {
574 if (verbose) {
575 dataLog(" - ", cachedRecovery->recovery(), " writes to NEW ", reg,
576 " but is also needed in an FPR.\n");
577 }
578 continue;
579 }
580 if (!tryWrites(*cachedRecovery)) {
581 if (verbose)
582 dataLog(" - Unable to write to NEW ", reg, " from ", cachedRecovery->recovery(), "\n");
583 failures.append(reg);
584 }
585 didProgress = true;
586 }
587 end = firstSafe;
588
589 // If we have cachedRecoveries that failed to write, it is
590 // because they are on the stack and we didn't have enough
591 // registers available at the time to load them into. If
592 // we have a free register, we should try again because it
593 // could free up some danger slots.
594 if (didProgress && hasFreeRegister()) {
595 Vector<VirtualRegister> stillFailing;
596 for (VirtualRegister failed : failures) {
597 CachedRecovery* cachedRecovery = getNew(failed);
598 // It could have been handled later if it had
599 // several targets
600 if (!cachedRecovery)
601 continue;
602
603 ASSERT(hasOnlySafeWrites(*cachedRecovery)
604 && !cachedRecovery->wantedJSValueRegs()
605 && cachedRecovery->wantedFPR() == InvalidFPRReg);
606 if (!tryWrites(*cachedRecovery))
607 stillFailing.append(failed);
608 }
609 failures = WTFMove(stillFailing);
610 }
611 if (verbose && firstSafe != dangerFrontier() + 1)
612 dataLog(" We freed up danger slots!\n");
613 } while (firstSafe != dangerFrontier() + 1);
614
615 return failures.isEmpty();
616}
617
618void CallFrameShuffler::prepareAny()
619{
620 ASSERT(!isUndecided());
621
622 updateDangerFrontier();
623
624 // First, we try to store any value that goes above the danger
625 // frontier. This will never use more registers since we are only
626 // loading+storing if we ensure that any register used for the load
627 // will be freed up after the stores (i.e., all stores are above
628 // the danger frontier, and there is no wanted register).
629 performSafeWrites();
630
631 // At this point, we couldn't have more available registers than
632 // we have withouth spilling: all values currently in registers
633 // either require a write to the danger zone, or have a wanted
634 // register, which means that in any case they will have to go
635 // through registers again.
636
637 // We now slowly free up the danger zone by first loading the old
638 // value on the danger frontier, spilling as many registers as
639 // needed to do so and ensuring that the corresponding slot in the
640 // new frame is now ready to be written. Then, we store the old
641 // value to its target location if possible (we could have failed
642 // to load it previously due to high pressure). Finally, we write
643 // to any of the newly safe slots that we can, which could free up
644 // registers (hence why we do it eagerly).
645 for (VirtualRegister reg = dangerFrontier(); reg >= firstNew(); reg -= 1) {
646 if (reg == dangerFrontier()) {
647 if (verbose)
648 dataLog(" Next slot (NEW ", reg, ") is the danger frontier\n");
649 CachedRecovery* cachedRecovery { getOld(newAsOld(dangerFrontier())) };
650 ASSERT(cachedRecovery);
651 ensureLoad(*cachedRecovery);
652 emitLoad(*cachedRecovery);
653 ensureBox(*cachedRecovery);
654 emitBox(*cachedRecovery);
655 if (hasOnlySafeWrites(*cachedRecovery))
656 tryWrites(*cachedRecovery);
657 } else if (verbose)
658 dataLog(" Next slot is NEW ", reg, "\n");
659
660 ASSERT(!isDangerNew(reg));
661 CachedRecovery* cachedRecovery = getNew(reg);
662 // This could be one of the header slots we don't care about.
663 if (!cachedRecovery) {
664 if (verbose)
665 dataLog(" + ", reg, " is OK\n");
666 continue;
667 }
668
669 if (canLoadAndBox(*cachedRecovery) && hasOnlySafeWrites(*cachedRecovery)
670 && !cachedRecovery->wantedJSValueRegs()
671 && cachedRecovery->wantedFPR() == InvalidFPRReg) {
672 emitLoad(*cachedRecovery);
673 emitBox(*cachedRecovery);
674 bool writesOK = tryWrites(*cachedRecovery);
675 ASSERT_UNUSED(writesOK, writesOK);
676 } else if (verbose)
677 dataLog(" - ", cachedRecovery->recovery(), " can't be handled just yet.\n");
678 }
679 ASSERT(dangerFrontier() < firstNew());
680
681 // Now, the danger zone is empty, but we still have a couple of
682 // things to do:
683 //
684 // 1) There could be remaining safe writes that failed earlier due
685 // to high register pressure and had nothing to do with the
686 // danger zone whatsoever.
687 //
688 // 2) Some wanted registers could have to be loaded (this could
689 // happen either when making a call to a new function with a
690 // lower number of arguments - since above here, we only load
691 // wanted registers when they are at the danger frontier -, or
692 // if a wanted register got spilled).
693 //
694 // 3) Some wanted registers could have been loaded in the wrong
695 // registers
696 //
697 // 4) We have to take care of some bookkeeping - namely, storing
698 // the argument count and updating the stack pointer.
699
700 // At this point, we must have enough registers available for
701 // handling 1). None of the loads can fail because we have been
702 // eagerly freeing up registers in all the previous phases - so
703 // the only values that are in registers at this point must have
704 // wanted registers.
705 if (verbose)
706 dataLog(" Danger zone is clear, performing remaining writes.\n");
707 for (VirtualRegister reg = firstNew(); reg <= lastNew(); reg += 1) {
708 CachedRecovery* cachedRecovery { getNew(reg) };
709 if (!cachedRecovery)
710 continue;
711
712 emitLoad(*cachedRecovery);
713 emitBox(*cachedRecovery);
714 bool writesOK = tryWrites(*cachedRecovery);
715 ASSERT_UNUSED(writesOK, writesOK);
716 }
717
718#if USE(JSVALUE64)
719 if (m_tagTypeNumber != InvalidGPRReg && m_newRegisters[m_tagTypeNumber])
720 releaseGPR(m_tagTypeNumber);
721#endif
722
723 // Handle 2) by loading all registers. We don't have to do any
724 // writes, since they have been taken care of above.
725 if (verbose)
726 dataLog(" Loading wanted registers into registers\n");
727 for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
728 CachedRecovery* cachedRecovery { m_newRegisters[reg] };
729 if (!cachedRecovery)
730 continue;
731
732 emitLoad(*cachedRecovery);
733 emitBox(*cachedRecovery);
734 ASSERT(cachedRecovery->targets().isEmpty());
735 }
736
737#if USE(JSVALUE64)
738 if (m_tagTypeNumber != InvalidGPRReg)
739 releaseGPR(m_tagTypeNumber);
740#endif
741
742 // At this point, we have read everything we cared about from the
743 // stack, and written everything we had to to the stack.
744 if (verbose)
745 dataLog(" Callee frame is fully set up\n");
746 if (!ASSERT_DISABLED) {
747 for (VirtualRegister reg = firstNew(); reg <= lastNew(); reg += 1)
748 ASSERT_UNUSED(reg, !getNew(reg));
749
750 for (CachedRecovery* cachedRecovery : m_cachedRecoveries) {
751 ASSERT_UNUSED(cachedRecovery, cachedRecovery->targets().isEmpty());
752 ASSERT(!cachedRecovery->recovery().isInJSStack());
753 }
754 }
755
756 // We need to handle 4) first because it implies releasing
757 // m_newFrameBase, which could be a wanted register.
758 if (verbose)
759 dataLog(" * Storing the argument count into ", VirtualRegister { CallFrameSlot::argumentCount }, "\n");
760 m_jit.store32(MacroAssembler::TrustedImm32(0),
761 addressForNew(VirtualRegister { CallFrameSlot::argumentCount }).withOffset(TagOffset));
762 RELEASE_ASSERT(m_numPassedArgs != UINT_MAX);
763 m_jit.store32(MacroAssembler::TrustedImm32(m_numPassedArgs),
764 addressForNew(VirtualRegister { CallFrameSlot::argumentCount }).withOffset(PayloadOffset));
765
766 if (!isSlowPath()) {
767 ASSERT(m_newFrameBase != MacroAssembler::stackPointerRegister);
768 if (verbose)
769 dataLog(" Releasing the new frame base pointer\n");
770 m_jit.move(m_newFrameBase, MacroAssembler::stackPointerRegister);
771 releaseGPR(m_newFrameBase);
772 }
773
774 // Finally we handle 3)
775 if (verbose)
776 dataLog(" Ensuring wanted registers are in the right register\n");
777 for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
778 CachedRecovery* cachedRecovery { m_newRegisters[reg] };
779 if (!cachedRecovery)
780 continue;
781
782 emitDisplace(*cachedRecovery);
783 }
784}
785
786} // namespace JSC
787
788#endif // ENABLE(JIT)
789