1/*
2 * Copyright (C) 2011-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "DFGOSREntry.h"
28
29#if ENABLE(DFG_JIT)
30
31#include "CallFrame.h"
32#include "CodeBlock.h"
33#include "DFGJITCode.h"
34#include "DFGNode.h"
35#include "InterpreterInlines.h"
36#include "JIT.h"
37#include "JSCInlines.h"
38#include "VMInlines.h"
39#include <wtf/CommaPrinter.h>
40
41namespace JSC { namespace DFG {
42
43void OSREntryData::dumpInContext(PrintStream& out, DumpContext* context) const
44{
45 out.print("bc#", m_bytecodeIndex, ", machine code = ", RawPointer(m_machineCode.executableAddress()));
46 out.print(", stack rules = [");
47
48 auto printOperand = [&] (VirtualRegister reg) {
49 out.print(inContext(m_expectedValues.operand(reg), context), " (");
50 VirtualRegister toReg;
51 bool overwritten = false;
52 for (OSREntryReshuffling reshuffling : m_reshufflings) {
53 if (reg == VirtualRegister(reshuffling.fromOffset)) {
54 toReg = VirtualRegister(reshuffling.toOffset);
55 break;
56 }
57 if (reg == VirtualRegister(reshuffling.toOffset))
58 overwritten = true;
59 }
60 if (!overwritten && !toReg.isValid())
61 toReg = reg;
62 if (toReg.isValid()) {
63 if (toReg.isLocal() && !m_machineStackUsed.get(toReg.toLocal()))
64 out.print("ignored");
65 else
66 out.print("maps to ", toReg);
67 } else
68 out.print("overwritten");
69 if (reg.isLocal() && m_localsForcedDouble.get(reg.toLocal()))
70 out.print(", forced double");
71 if (reg.isLocal() && m_localsForcedAnyInt.get(reg.toLocal()))
72 out.print(", forced machine int");
73 out.print(")");
74 };
75
76 CommaPrinter comma;
77 for (size_t argumentIndex = m_expectedValues.numberOfArguments(); argumentIndex--;) {
78 out.print(comma, "arg", argumentIndex, ":");
79 printOperand(virtualRegisterForArgument(argumentIndex));
80 }
81 for (size_t localIndex = 0; localIndex < m_expectedValues.numberOfLocals(); ++localIndex) {
82 out.print(comma, "loc", localIndex, ":");
83 printOperand(virtualRegisterForLocal(localIndex));
84 }
85
86 out.print("], machine stack used = ", m_machineStackUsed);
87}
88
89void OSREntryData::dump(PrintStream& out) const
90{
91 dumpInContext(out, nullptr);
92}
93
94SUPPRESS_ASAN
95void* prepareOSREntry(ExecState* exec, CodeBlock* codeBlock, unsigned bytecodeIndex)
96{
97 ASSERT(JITCode::isOptimizingJIT(codeBlock->jitType()));
98 ASSERT(codeBlock->alternative());
99 ASSERT(codeBlock->alternative()->jitType() == JITType::BaselineJIT);
100 ASSERT(!codeBlock->jitCodeMap());
101 ASSERT(codeBlock->jitCode()->dfgCommon()->isStillValid);
102
103 if (!Options::useOSREntryToDFG())
104 return nullptr;
105
106 if (Options::verboseOSR()) {
107 dataLog(
108 "DFG OSR in ", *codeBlock->alternative(), " -> ", *codeBlock,
109 " from bc#", bytecodeIndex, "\n");
110 }
111
112 VM* vm = &exec->vm();
113
114 sanitizeStackForVM(vm);
115
116 if (bytecodeIndex)
117 codeBlock->ownerExecutable()->setDidTryToEnterInLoop(true);
118
119 if (codeBlock->jitType() != JITType::DFGJIT) {
120 RELEASE_ASSERT(codeBlock->jitType() == JITType::FTLJIT);
121
122 // When will this happen? We could have:
123 //
124 // - An exit from the FTL JIT into the baseline JIT followed by an attempt
125 // to reenter. We're fine with allowing this to fail. If it happens
126 // enough we'll just reoptimize. It basically means that the OSR exit cost
127 // us dearly and so reoptimizing is the right thing to do.
128 //
129 // - We have recursive code with hot loops. Consider that foo has a hot loop
130 // that calls itself. We have two foo's on the stack, lets call them foo1
131 // and foo2, with foo1 having called foo2 from foo's hot loop. foo2 gets
132 // optimized all the way into the FTL. Then it returns into foo1, and then
133 // foo1 wants to get optimized. It might reach this conclusion from its
134 // hot loop and attempt to OSR enter. And we'll tell it that it can't. It
135 // might be worth addressing this case, but I just think this case will
136 // be super rare. For now, if it does happen, it'll cause some compilation
137 // thrashing.
138
139 if (Options::verboseOSR())
140 dataLog(" OSR failed because the target code block is not DFG.\n");
141 return nullptr;
142 }
143
144 JITCode* jitCode = codeBlock->jitCode()->dfg();
145 OSREntryData* entry = jitCode->osrEntryDataForBytecodeIndex(bytecodeIndex);
146
147 if (!entry) {
148 if (Options::verboseOSR())
149 dataLogF(" OSR failed because the entrypoint was optimized out.\n");
150 return nullptr;
151 }
152
153 ASSERT(entry->m_bytecodeIndex == bytecodeIndex);
154
155 // The code below checks if it is safe to perform OSR entry. It may find
156 // that it is unsafe to do so, for any number of reasons, which are documented
157 // below. If the code decides not to OSR then it returns 0, and it's the caller's
158 // responsibility to patch up the state in such a way as to ensure that it's
159 // both safe and efficient to continue executing baseline code for now. This
160 // should almost certainly include calling either codeBlock->optimizeAfterWarmUp()
161 // or codeBlock->dontOptimizeAnytimeSoon().
162
163 // 1) Verify predictions. If the predictions are inconsistent with the actual
164 // values, then OSR entry is not possible at this time. It's tempting to
165 // assume that we could somehow avoid this case. We can certainly avoid it
166 // for first-time loop OSR - that is, OSR into a CodeBlock that we have just
167 // compiled. Then we are almost guaranteed that all of the predictions will
168 // check out. It would be pretty easy to make that a hard guarantee. But
169 // then there would still be the case where two call frames with the same
170 // baseline CodeBlock are on the stack at the same time. The top one
171 // triggers compilation and OSR. In that case, we may no longer have
172 // accurate value profiles for the one deeper in the stack. Hence, when we
173 // pop into the CodeBlock that is deeper on the stack, we might OSR and
174 // realize that the predictions are wrong. Probably, in most cases, this is
175 // just an anomaly in the sense that the older CodeBlock simply went off
176 // into a less-likely path. So, the wisest course of action is to simply not
177 // OSR at this time.
178
179 for (size_t argument = 0; argument < entry->m_expectedValues.numberOfArguments(); ++argument) {
180 if (argument >= exec->argumentCountIncludingThis()) {
181 if (Options::verboseOSR()) {
182 dataLogF(" OSR failed because argument %zu was not passed, expected ", argument);
183 entry->m_expectedValues.argument(argument).dump(WTF::dataFile());
184 dataLogF(".\n");
185 }
186 return nullptr;
187 }
188
189 JSValue value;
190 if (!argument)
191 value = exec->thisValue();
192 else
193 value = exec->argument(argument - 1);
194
195 if (!entry->m_expectedValues.argument(argument).validateOSREntryValue(value, FlushedJSValue)) {
196 if (Options::verboseOSR()) {
197 dataLog(
198 " OSR failed because argument ", argument, " is ", value,
199 ", expected ", entry->m_expectedValues.argument(argument), ".\n");
200 }
201 return nullptr;
202 }
203 }
204
205 for (size_t local = 0; local < entry->m_expectedValues.numberOfLocals(); ++local) {
206 int localOffset = virtualRegisterForLocal(local).offset();
207 JSValue value = exec->registers()[localOffset].asanUnsafeJSValue();
208 FlushFormat format = FlushedJSValue;
209
210 if (entry->m_localsForcedAnyInt.get(local)) {
211 if (!value.isAnyInt()) {
212 dataLogLnIf(Options::verboseOSR(),
213 " OSR failed because variable ", localOffset, " is ",
214 value, ", expected ",
215 "machine int.");
216 return nullptr;
217 }
218 value = jsDoubleNumber(value.asAnyInt());
219 format = FlushedInt52;
220 }
221
222 if (entry->m_localsForcedDouble.get(local)) {
223 if (!value.isNumber()) {
224 dataLogLnIf(Options::verboseOSR(),
225 " OSR failed because variable ", localOffset, " is ",
226 value, ", expected number.");
227 return nullptr;
228 }
229 value = jsDoubleNumber(value.asNumber());
230 format = FlushedDouble;
231 }
232
233 if (!entry->m_expectedValues.local(local).validateOSREntryValue(value, format)) {
234 dataLogLnIf(Options::verboseOSR(),
235 " OSR failed because variable ", VirtualRegister(localOffset), " is ",
236 value, ", expected ",
237 entry->m_expectedValues.local(local), ".");
238 return nullptr;
239 }
240 }
241
242 // 2) Check the stack height. The DFG JIT may require a taller stack than the
243 // baseline JIT, in some cases. If we can't grow the stack, then don't do
244 // OSR right now. That's the only option we have unless we want basic block
245 // boundaries to start throwing RangeErrors. Although that would be possible,
246 // it seems silly: you'd be diverting the program to error handling when it
247 // would have otherwise just kept running albeit less quickly.
248
249 unsigned frameSizeForCheck = jitCode->common.requiredRegisterCountForExecutionAndExit();
250 if (UNLIKELY(!vm->ensureStackCapacityFor(&exec->registers()[virtualRegisterForLocal(frameSizeForCheck - 1).offset()]))) {
251 if (Options::verboseOSR())
252 dataLogF(" OSR failed because stack growth failed.\n");
253 return nullptr;
254 }
255
256 if (Options::verboseOSR())
257 dataLogF(" OSR should succeed.\n");
258
259 // At this point we're committed to entering. We will do some work to set things up,
260 // but we also rely on our caller recognizing that when we return a non-null pointer,
261 // that means that we're already past the point of no return and we must succeed at
262 // entering.
263
264 // 3) Set up the data in the scratch buffer and perform data format conversions.
265
266 unsigned frameSize = jitCode->common.frameRegisterCount;
267 unsigned baselineFrameSize = entry->m_expectedValues.numberOfLocals();
268 unsigned maxFrameSize = std::max(frameSize, baselineFrameSize);
269
270 Register* scratch = bitwise_cast<Register*>(vm->scratchBufferForSize(sizeof(Register) * (2 + CallFrame::headerSizeInRegisters + maxFrameSize))->dataBuffer());
271
272 *bitwise_cast<size_t*>(scratch + 0) = frameSize;
273
274 void* targetPC = entry->m_machineCode.executableAddress();
275 RELEASE_ASSERT(codeBlock->jitCode()->contains(entry->m_machineCode.untaggedExecutableAddress()));
276 if (Options::verboseOSR())
277 dataLogF(" OSR using target PC %p.\n", targetPC);
278 RELEASE_ASSERT(targetPC);
279 *bitwise_cast<void**>(scratch + 1) = retagCodePtr(targetPC, OSREntryPtrTag, bitwise_cast<PtrTag>(exec));
280
281 Register* pivot = scratch + 2 + CallFrame::headerSizeInRegisters;
282
283 for (int index = -CallFrame::headerSizeInRegisters; index < static_cast<int>(baselineFrameSize); ++index) {
284 VirtualRegister reg(-1 - index);
285
286 if (reg.isLocal()) {
287 if (entry->m_localsForcedDouble.get(reg.toLocal())) {
288 *bitwise_cast<double*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asNumber();
289 continue;
290 }
291
292 if (entry->m_localsForcedAnyInt.get(reg.toLocal())) {
293 *bitwise_cast<int64_t*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asAnyInt() << JSValue::int52ShiftAmount;
294 continue;
295 }
296 }
297
298 pivot[index] = exec->registers()[reg.offset()].asanUnsafeJSValue();
299 }
300
301 // 4) Reshuffle those registers that need reshuffling.
302 Vector<JSValue> temporaryLocals(entry->m_reshufflings.size());
303 for (unsigned i = entry->m_reshufflings.size(); i--;)
304 temporaryLocals[i] = pivot[VirtualRegister(entry->m_reshufflings[i].fromOffset).toLocal()].asanUnsafeJSValue();
305 for (unsigned i = entry->m_reshufflings.size(); i--;)
306 pivot[VirtualRegister(entry->m_reshufflings[i].toOffset).toLocal()] = temporaryLocals[i];
307
308 // 5) Clear those parts of the call frame that the DFG ain't using. This helps GC on
309 // some programs by eliminating some stale pointer pathologies.
310 for (unsigned i = frameSize; i--;) {
311 if (entry->m_machineStackUsed.get(i))
312 continue;
313 pivot[i] = JSValue();
314 }
315
316 // 6) Copy our callee saves to buffer.
317#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
318 const RegisterAtOffsetList* registerSaveLocations = codeBlock->calleeSaveRegisters();
319 RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
320 RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs());
321
322 unsigned registerCount = registerSaveLocations->size();
323 VMEntryRecord* record = vmEntryRecord(vm->topEntryFrame);
324 for (unsigned i = 0; i < registerCount; i++) {
325 RegisterAtOffset currentEntry = registerSaveLocations->at(i);
326 if (dontSaveRegisters.get(currentEntry.reg()))
327 continue;
328 RegisterAtOffset* calleeSavesEntry = allCalleeSaves->find(currentEntry.reg());
329
330 *(bitwise_cast<intptr_t*>(pivot - 1) - currentEntry.offsetAsIndex()) = record->calleeSaveRegistersBuffer[calleeSavesEntry->offsetAsIndex()];
331 }
332#endif
333
334 // 7) Fix the call frame to have the right code block.
335
336 *bitwise_cast<CodeBlock**>(pivot - 1 - CallFrameSlot::codeBlock) = codeBlock;
337
338 if (Options::verboseOSR())
339 dataLogF(" OSR returning data buffer %p.\n", scratch);
340 return scratch;
341}
342
343MacroAssemblerCodePtr<ExceptionHandlerPtrTag> prepareCatchOSREntry(ExecState* exec, CodeBlock* codeBlock, unsigned bytecodeIndex)
344{
345 ASSERT(codeBlock->jitType() == JITType::DFGJIT || codeBlock->jitType() == JITType::FTLJIT);
346 ASSERT(codeBlock->jitCode()->dfgCommon()->isStillValid);
347
348 if (!Options::useOSREntryToDFG() && codeBlock->jitCode()->jitType() == JITType::DFGJIT)
349 return nullptr;
350 if (!Options::useOSREntryToFTL() && codeBlock->jitCode()->jitType() == JITType::FTLJIT)
351 return nullptr;
352
353 VM& vm = exec->vm();
354
355 CommonData* dfgCommon = codeBlock->jitCode()->dfgCommon();
356 RELEASE_ASSERT(dfgCommon);
357 DFG::CatchEntrypointData* catchEntrypoint = dfgCommon->catchOSREntryDataForBytecodeIndex(bytecodeIndex);
358 if (!catchEntrypoint) {
359 // This can be null under some circumstances. The most common is that we didn't
360 // compile this op_catch as an entrypoint since it had never executed when starting
361 // the compilation.
362 return nullptr;
363 }
364
365 // We're only allowed to OSR enter if we've proven we have compatible argument types.
366 for (unsigned argument = 0; argument < catchEntrypoint->argumentFormats.size(); ++argument) {
367 JSValue value = exec->uncheckedR(virtualRegisterForArgument(argument)).jsValue();
368 switch (catchEntrypoint->argumentFormats[argument]) {
369 case DFG::FlushedInt32:
370 if (!value.isInt32())
371 return nullptr;
372 break;
373 case DFG::FlushedCell:
374 if (!value.isCell())
375 return nullptr;
376 break;
377 case DFG::FlushedBoolean:
378 if (!value.isBoolean())
379 return nullptr;
380 break;
381 case DFG::DeadFlush:
382 // This means the argument is not alive. Therefore, it's allowed to be any type.
383 break;
384 case DFG::FlushedJSValue:
385 // An argument is trivially a JSValue.
386 break;
387 default:
388 RELEASE_ASSERT_NOT_REACHED();
389 }
390 }
391
392 unsigned frameSizeForCheck = dfgCommon->requiredRegisterCountForExecutionAndExit();
393 if (UNLIKELY(!vm.ensureStackCapacityFor(&exec->registers()[virtualRegisterForLocal(frameSizeForCheck).offset()])))
394 return nullptr;
395
396 auto instruction = exec->codeBlock()->instructions().at(exec->bytecodeOffset());
397 ASSERT(instruction->is<OpCatch>());
398 ValueProfileAndOperandBuffer* buffer = instruction->as<OpCatch>().metadata(exec).m_buffer;
399 JSValue* dataBuffer = reinterpret_cast<JSValue*>(dfgCommon->catchOSREntryBuffer->dataBuffer());
400 unsigned index = 0;
401 buffer->forEach([&] (ValueProfileAndOperand& profile) {
402 if (!VirtualRegister(profile.m_operand).isLocal())
403 return;
404 dataBuffer[index] = exec->uncheckedR(profile.m_operand).jsValue();
405 ++index;
406 });
407
408 // The active length of catchOSREntryBuffer will be zeroed by ClearCatchLocals node.
409 dfgCommon->catchOSREntryBuffer->setActiveLength(sizeof(JSValue) * index);
410 return catchEntrypoint->machineCode;
411}
412
413} } // namespace JSC::DFG
414
415#endif // ENABLE(DFG_JIT)
416