1/*
2 * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmB3IRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AllowMacroScratchRegisterUsageIf.h"
32#include "B3BasicBlockInlines.h"
33#include "B3CCallValue.h"
34#include "B3Compile.h"
35#include "B3ConstPtrValue.h"
36#include "B3FixSSA.h"
37#include "B3Generate.h"
38#include "B3InsertionSet.h"
39#include "B3SlotBaseValue.h"
40#include "B3StackmapGenerationParams.h"
41#include "B3SwitchValue.h"
42#include "B3UpsilonValue.h"
43#include "B3Validate.h"
44#include "B3ValueInlines.h"
45#include "B3ValueKey.h"
46#include "B3Variable.h"
47#include "B3VariableValue.h"
48#include "B3WasmAddressValue.h"
49#include "B3WasmBoundsCheckValue.h"
50#include "DisallowMacroScratchRegisterUsage.h"
51#include "JSCInlines.h"
52#include "ScratchRegisterAllocator.h"
53#include "VirtualRegister.h"
54#include "WasmCallingConvention.h"
55#include "WasmContextInlines.h"
56#include "WasmExceptionType.h"
57#include "WasmFunctionParser.h"
58#include "WasmInstance.h"
59#include "WasmMemory.h"
60#include "WasmOMGPlan.h"
61#include "WasmOpcodeOrigin.h"
62#include "WasmSignatureInlines.h"
63#include "WasmThunks.h"
64#include <limits>
65#include <wtf/Optional.h>
66#include <wtf/StdLibExtras.h>
67
68void dumpProcedure(void* ptr)
69{
70 JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
71 proc->dump(WTF::dataFile());
72}
73
74namespace JSC { namespace Wasm {
75
76using namespace B3;
77
78namespace {
79namespace WasmB3IRGeneratorInternal {
80static const bool verbose = false;
81}
82}
83
84class B3IRGenerator {
85public:
86 struct ControlData {
87 ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
88 : blockType(type)
89 , continuation(continuation)
90 , special(special)
91 {
92 if (signature != Void)
93 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
94 }
95
96 ControlData()
97 {
98 }
99
100 void dump(PrintStream& out) const
101 {
102 switch (type()) {
103 case BlockType::If:
104 out.print("If: ");
105 break;
106 case BlockType::Block:
107 out.print("Block: ");
108 break;
109 case BlockType::Loop:
110 out.print("Loop: ");
111 break;
112 case BlockType::TopLevel:
113 out.print("TopLevel: ");
114 break;
115 }
116 out.print("Continuation: ", *continuation, ", Special: ");
117 if (special)
118 out.print(*special);
119 else
120 out.print("None");
121 }
122
123 BlockType type() const { return blockType; }
124
125 bool hasNonVoidSignature() const { return result.size(); }
126
127 BasicBlock* targetBlockForBranch()
128 {
129 if (type() == BlockType::Loop)
130 return special;
131 return continuation;
132 }
133
134 void convertIfToBlock()
135 {
136 ASSERT(type() == BlockType::If);
137 blockType = BlockType::Block;
138 special = nullptr;
139 }
140
141 using ResultList = Vector<Value*, 1>; // Value must be a Phi
142
143 ResultList resultForBranch() const
144 {
145 if (type() == BlockType::Loop)
146 return ResultList();
147 return result;
148 }
149
150 private:
151 friend class B3IRGenerator;
152 BlockType blockType;
153 BasicBlock* continuation;
154 BasicBlock* special;
155 ResultList result;
156 };
157
158 typedef Value* ExpressionType;
159 typedef ControlData ControlType;
160 typedef Vector<ExpressionType, 1> ExpressionList;
161 typedef ControlData::ResultList ResultList;
162 typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
163
164 static constexpr ExpressionType emptyExpression() { return nullptr; }
165
166 typedef String ErrorType;
167 typedef Unexpected<ErrorType> UnexpectedResult;
168 typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
169 typedef Expected<void, ErrorType> PartialResult;
170 template <typename ...Args>
171 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
172 {
173 using namespace FailureHelper; // See ADL comment in WasmParser.h.
174 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
175 }
176#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
177 if (UNLIKELY(condition)) \
178 return fail(__VA_ARGS__); \
179 } while (0)
180
181 B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
182
183 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
184 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
185 ExpressionType addConstant(Type, uint64_t);
186
187 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
188
189 // Locals
190 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
191 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
192
193 // Globals
194 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
195 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
196
197 // Memory
198 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
199 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
200 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
201 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
202
203 // Basic operators
204 template<OpType>
205 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
206 template<OpType>
207 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
208 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
209
210 // Control flow
211 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
212 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
213 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
214 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
215 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
216 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
217
218 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
219 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
220 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
221 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
222 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
223
224 // Calls
225 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
226 PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
227 PartialResult WARN_UNUSED_RETURN addUnreachable();
228
229 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
230 void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
231
232 Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
233 void insertConstants();
234
235 ALWAYS_INLINE void didKill(ExpressionType) { }
236
237private:
238 void emitExceptionCheck(CCallHelpers&, ExceptionType);
239
240 void emitTierUpCheck(uint32_t decrementCount, Origin);
241
242 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
243 B3::Kind memoryKind(B3::Opcode memoryOp);
244 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
245 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
246
247 void unify(const ExpressionType phi, const ExpressionType source);
248 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
249
250 void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
251
252 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
253
254 void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
255 enum class RestoreCachedStackLimit { No, Yes };
256 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
257
258 Origin origin();
259
260 FunctionParser<B3IRGenerator>* m_parser { nullptr };
261 const ModuleInformation& m_info;
262 const MemoryMode m_mode { MemoryMode::BoundsChecking };
263 const CompilationMode m_compilationMode { CompilationMode::BBQMode };
264 const unsigned m_functionIndex { UINT_MAX };
265 const TierUpCount* m_tierUp { nullptr };
266
267 Procedure& m_proc;
268 BasicBlock* m_currentBlock { nullptr };
269 Vector<Variable*> m_locals;
270 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
271 HashMap<ValueKey, Value*> m_constantPool;
272 InsertionSet m_constantInsertionValues;
273 GPRReg m_memoryBaseGPR { InvalidGPRReg };
274 GPRReg m_memorySizeGPR { InvalidGPRReg };
275 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
276 bool m_makesCalls { false };
277
278 Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
279 bool m_usesInstanceValue { false };
280 Value* instanceValue()
281 {
282 m_usesInstanceValue = true;
283 return m_instanceValue;
284 }
285
286 uint32_t m_maxNumJSCallArguments { 0 };
287};
288
289// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
290int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
291{
292 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
293 ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
294 return 0;
295 }
296 return offset;
297}
298
299void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
300{
301 if (Context::useFastTLS()) {
302 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
303 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
304 patchpoint->clobber(RegisterSet::macroScratchRegisters());
305 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
306 patchpoint->setGenerator(
307 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
308 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
309 jit.storeWasmContextInstance(params[0].gpr());
310 });
311 return;
312 }
313
314 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
315 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
316 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
317 Effects effects = Effects::none();
318 effects.writesPinned = true;
319 effects.reads = B3::HeapRange::top();
320 patchpoint->effects = effects;
321 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
322 patchpoint->append(arg, ValueRep::SomeRegister);
323 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
324 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
325 jit.move(param[0].gpr(), wasmContextInstanceGPR);
326 });
327}
328
329B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
330 : m_info(info)
331 , m_mode(mode)
332 , m_compilationMode(compilationMode)
333 , m_functionIndex(functionIndex)
334 , m_tierUp(tierUp)
335 , m_proc(procedure)
336 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
337 , m_constantInsertionValues(m_proc)
338{
339 m_currentBlock = m_proc.addBlock();
340
341 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
342 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
343
344 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
345 m_proc.pinRegister(m_memoryBaseGPR);
346
347 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
348 if (!Context::useFastTLS())
349 m_proc.pinRegister(m_wasmContextInstanceGPR);
350
351 if (mode != MemoryMode::Signaling) {
352 m_memorySizeGPR = pinnedRegs.sizeRegister;
353 m_proc.pinRegister(m_memorySizeGPR);
354 }
355
356 if (throwWasmException)
357 Thunks::singleton().setThrowWasmException(throwWasmException);
358
359 if (info.memory) {
360 m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
361 AllowMacroScratchRegisterUsage allowScratch(jit);
362 switch (m_mode) {
363 case MemoryMode::BoundsChecking:
364 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
365 break;
366 case MemoryMode::Signaling:
367 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
368 break;
369 }
370 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
371 });
372
373 switch (m_mode) {
374 case MemoryMode::BoundsChecking:
375 break;
376 case MemoryMode::Signaling:
377 // Most memory accesses in signaling mode don't do an explicit
378 // exception check because they can rely on fault handling to detect
379 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
380 // thunk to exist so that it can jump to that thunk.
381 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
382 CRASH();
383 break;
384 }
385 }
386
387 wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
388
389 {
390 B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
391 B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
392 m_instanceValue = stackOverflowCheck;
393 stackOverflowCheck->appendSomeRegister(framePointer);
394 stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
395 if (!Context::useFastTLS()) {
396 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
397 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
398 stackOverflowCheck->effects.writesPinned = false;
399 stackOverflowCheck->effects.readsPinned = true;
400 stackOverflowCheck->resultConstraint = ValueRep::reg(m_wasmContextInstanceGPR);
401 }
402 stackOverflowCheck->numGPScratchRegisters = 2;
403 stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
404 const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
405 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
406 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
407 // This allows us to elide stack checks for functions that are terminal nodes in the call
408 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
409 // having any such terminal node have its parent caller include some extra size in its
410 // own check for it. The goal here is twofold:
411 // 1. Emit less code.
412 // 2. Try to speed things up by skipping stack checks.
413 minimumParentCheckSize,
414 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
415 // spill all arguments to the stack, we ensure that a stack check here covers the
416 // stack that such a stub would use.
417 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
418 ));
419 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
420 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
421 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
422
423 GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
424
425 // This allows leaf functions to not do stack checks if their frame size is within
426 // certain limits since their caller would have already done the check.
427 if (needsOverflowCheck) {
428 AllowMacroScratchRegisterUsage allowScratch(jit);
429 GPRReg fp = params[1].gpr();
430 GPRReg scratch1 = params.gpScratch(0);
431 GPRReg scratch2 = params.gpScratch(1);
432
433 if (Context::useFastTLS())
434 jit.loadWasmContextInstance(contextInstance);
435
436 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
437 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
438 MacroAssembler::JumpList overflow;
439 if (UNLIKELY(needUnderflowCheck))
440 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
441 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
442 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
443 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
444 });
445 } else if (m_usesInstanceValue && Context::useFastTLS()) {
446 // No overflow check is needed, but the instance values still needs to be correct.
447 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
448 jit.loadWasmContextInstance(contextInstance);
449 } else {
450 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
451 }
452 });
453 }
454
455 emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
456}
457
458void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
459{
460 restoreWasmContextInstance(proc, block, instance);
461
462 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
463 // The Instance caches the stack limit, but also knows where its canonical location is.
464 Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
465 Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
466 block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
467 }
468
469 if (!!memory) {
470 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
471 RegisterSet clobbers;
472 clobbers.set(pinnedRegs->baseMemoryPointer);
473 clobbers.set(pinnedRegs->sizeRegister);
474 if (!isARM64())
475 clobbers.set(RegisterSet::macroScratchRegisters());
476
477 B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
478 Effects effects = Effects::none();
479 effects.writesPinned = true;
480 effects.reads = B3::HeapRange::top();
481 patchpoint->effects = effects;
482 patchpoint->clobber(clobbers);
483 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
484
485 patchpoint->append(instance, ValueRep::SomeRegister);
486 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
487 RELEASE_ASSERT(!Gigacage::isEnabled(Gigacage::Primitive) || !isARM64());
488 AllowMacroScratchRegisterUsageIf allowScratch(jit, !isARM64());
489 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
490 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
491
492 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
493 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
494
495 jit.cageConditionally(Gigacage::Primitive, baseMemory, scratchOrSize);
496 });
497 }
498}
499
500void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
501{
502 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
503 auto jumpToExceptionStub = jit.jump();
504
505 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
506 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
507 });
508}
509
510Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
511{
512 auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
513 Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
514 m_constantInsertionValues.insertValue(0, result);
515 return result;
516 });
517 return result.iterator->value;
518}
519
520void B3IRGenerator::insertConstants()
521{
522 m_constantInsertionValues.execute(m_proc.at(0));
523}
524
525auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
526{
527 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
528 totalBytesChecked += m_locals.size();
529 uint32_t totalBytes;
530 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
531
532 for (uint32_t i = 0; i < count; ++i) {
533 Variable* local = m_proc.addVariable(toB3Type(type));
534 m_locals.uncheckedAppend(local);
535 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), 0, Origin()));
536 }
537 return { };
538}
539
540auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
541{
542 ASSERT(!m_locals.size());
543 WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
544
545 m_locals.grow(signature.argumentCount());
546 wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
547 [=] (ExpressionType argument, unsigned i) {
548 Variable* argumentVariable = m_proc.addVariable(argument->type());
549 m_locals[i] = argumentVariable;
550 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
551 });
552 return { };
553}
554
555auto B3IRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
556{
557 result = m_currentBlock->appendNew<Value>(m_proc, B3::Equal, origin(), value, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), JSValue::encode(jsNull())));
558 return { };
559}
560
561auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
562{
563 ASSERT(m_locals[index]);
564 result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
565 return { };
566}
567
568auto B3IRGenerator::addUnreachable() -> PartialResult
569{
570 B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
571 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
572 this->emitExceptionCheck(jit, ExceptionType::Unreachable);
573 });
574 unreachable->effects.terminal = true;
575 return { };
576}
577
578auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
579{
580 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
581 instance->storeTopCallFrame(callFrame);
582
583 if (delta < 0)
584 return -1;
585
586 auto grown = instance->memory()->grow(PageCount(delta));
587 if (!grown) {
588 switch (grown.error()) {
589 case Memory::GrowFailReason::InvalidDelta:
590 case Memory::GrowFailReason::InvalidGrowSize:
591 case Memory::GrowFailReason::WouldExceedMaximum:
592 case Memory::GrowFailReason::OutOfMemory:
593 return -1;
594 }
595 RELEASE_ASSERT_NOT_REACHED();
596 }
597
598 return grown.value().pageCount();
599 };
600
601 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
602 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(growMemory, B3CCallPtrTag)),
603 m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
604
605 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
606
607 return { };
608}
609
610auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
611{
612 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
613 Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedMemorySize()));
614
615 constexpr uint32_t shiftValue = 16;
616 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
617 Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
618 size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
619
620 result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
621
622 return { };
623}
624
625auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
626{
627 ASSERT(m_locals[index]);
628 m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
629 return { };
630}
631
632auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
633{
634 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
635 result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
636 return { };
637}
638
639auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
640{
641 ASSERT(toB3Type(m_info.globals[index].type) == value->type());
642 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
643 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
644 return { };
645}
646
647inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
648{
649 ASSERT(m_memoryBaseGPR);
650
651 switch (m_mode) {
652 case MemoryMode::BoundsChecking: {
653 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
654 ASSERT(m_memorySizeGPR);
655 ASSERT(sizeOfOperation + offset > offset);
656 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
657 break;
658 }
659
660 case MemoryMode::Signaling: {
661 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
662 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
663 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
664 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
665 // register + large offset access. We don't think this will be generated frequently.
666 //
667 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
668 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
669 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
670 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
671 if (offset >= Memory::fastMappedRedzoneBytes()) {
672 size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
673 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
674 }
675 break;
676 }
677 }
678
679 pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
680 return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
681}
682
683inline uint32_t sizeOfLoadOp(LoadOpType op)
684{
685 switch (op) {
686 case LoadOpType::I32Load8S:
687 case LoadOpType::I32Load8U:
688 case LoadOpType::I64Load8S:
689 case LoadOpType::I64Load8U:
690 return 1;
691 case LoadOpType::I32Load16S:
692 case LoadOpType::I64Load16S:
693 case LoadOpType::I32Load16U:
694 case LoadOpType::I64Load16U:
695 return 2;
696 case LoadOpType::I32Load:
697 case LoadOpType::I64Load32S:
698 case LoadOpType::I64Load32U:
699 case LoadOpType::F32Load:
700 return 4;
701 case LoadOpType::I64Load:
702 case LoadOpType::F64Load:
703 return 8;
704 }
705 RELEASE_ASSERT_NOT_REACHED();
706}
707
708inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
709{
710 if (m_mode == MemoryMode::Signaling)
711 return trapping(memoryOp);
712 return memoryOp;
713}
714
715inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
716{
717 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
718
719 switch (op) {
720 case LoadOpType::I32Load8S: {
721 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
722 }
723
724 case LoadOpType::I64Load8S: {
725 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
726 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
727 }
728
729 case LoadOpType::I32Load8U: {
730 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
731 }
732
733 case LoadOpType::I64Load8U: {
734 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
735 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
736 }
737
738 case LoadOpType::I32Load16S: {
739 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
740 }
741
742 case LoadOpType::I64Load16S: {
743 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
744 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
745 }
746
747 case LoadOpType::I32Load16U: {
748 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
749 }
750
751 case LoadOpType::I64Load16U: {
752 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
753 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
754 }
755
756 case LoadOpType::I32Load: {
757 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
758 }
759
760 case LoadOpType::I64Load32U: {
761 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
762 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
763 }
764
765 case LoadOpType::I64Load32S: {
766 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
767 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
768 }
769
770 case LoadOpType::I64Load: {
771 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
772 }
773
774 case LoadOpType::F32Load: {
775 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
776 }
777
778 case LoadOpType::F64Load: {
779 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
780 }
781 }
782 RELEASE_ASSERT_NOT_REACHED();
783}
784
785auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
786{
787 ASSERT(pointer->type() == Int32);
788
789 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
790 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
791 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
792 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
793 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
794 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
795 });
796
797 switch (op) {
798 case LoadOpType::I32Load8S:
799 case LoadOpType::I32Load16S:
800 case LoadOpType::I32Load:
801 case LoadOpType::I32Load16U:
802 case LoadOpType::I32Load8U:
803 result = constant(Int32, 0);
804 break;
805 case LoadOpType::I64Load8S:
806 case LoadOpType::I64Load8U:
807 case LoadOpType::I64Load16S:
808 case LoadOpType::I64Load32U:
809 case LoadOpType::I64Load32S:
810 case LoadOpType::I64Load:
811 case LoadOpType::I64Load16U:
812 result = constant(Int64, 0);
813 break;
814 case LoadOpType::F32Load:
815 result = constant(Float, 0);
816 break;
817 case LoadOpType::F64Load:
818 result = constant(Double, 0);
819 break;
820 }
821
822 } else
823 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
824
825 return { };
826}
827
828inline uint32_t sizeOfStoreOp(StoreOpType op)
829{
830 switch (op) {
831 case StoreOpType::I32Store8:
832 case StoreOpType::I64Store8:
833 return 1;
834 case StoreOpType::I32Store16:
835 case StoreOpType::I64Store16:
836 return 2;
837 case StoreOpType::I32Store:
838 case StoreOpType::I64Store32:
839 case StoreOpType::F32Store:
840 return 4;
841 case StoreOpType::I64Store:
842 case StoreOpType::F64Store:
843 return 8;
844 }
845 RELEASE_ASSERT_NOT_REACHED();
846}
847
848
849inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
850{
851 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
852
853 switch (op) {
854 case StoreOpType::I64Store8:
855 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
856 FALLTHROUGH;
857
858 case StoreOpType::I32Store8:
859 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
860 return;
861
862 case StoreOpType::I64Store16:
863 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
864 FALLTHROUGH;
865
866 case StoreOpType::I32Store16:
867 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
868 return;
869
870 case StoreOpType::I64Store32:
871 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
872 FALLTHROUGH;
873
874 case StoreOpType::I64Store:
875 case StoreOpType::I32Store:
876 case StoreOpType::F32Store:
877 case StoreOpType::F64Store:
878 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
879 return;
880 }
881 RELEASE_ASSERT_NOT_REACHED();
882}
883
884auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
885{
886 ASSERT(pointer->type() == Int32);
887
888 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
889 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
890 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
891 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
892 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
893 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
894 });
895 } else
896 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
897
898 return { };
899}
900
901auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
902{
903 result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
904 return { };
905}
906
907B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
908{
909 return constant(toB3Type(type), value);
910}
911
912void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
913{
914 if (!m_tierUp)
915 return;
916
917 ASSERT(m_tierUp);
918 Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
919 Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
920 Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
921 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
922
923 PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
924 Effects effects = Effects::none();
925 // FIXME: we should have a more precise heap range for the tier up count.
926 effects.reads = B3::HeapRange::top();
927 effects.writes = B3::HeapRange::top();
928 patch->effects = effects;
929
930 patch->append(newCountDown, ValueRep::SomeRegister);
931 patch->append(oldCountDown, ValueRep::SomeRegister);
932 patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
933 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
934 MacroAssembler::Label tierUpResume = jit.label();
935
936 params.addLatePath([=] (CCallHelpers& jit) {
937 tierUp.link(&jit);
938
939 const unsigned extraPaddingBytes = 0;
940 RegisterSet registersToSpill = { };
941 registersToSpill.add(GPRInfo::argumentGPR1);
942 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
943
944 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
945 MacroAssembler::Call call = jit.nearCall();
946
947 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
948 jit.jump(tierUpResume);
949
950 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
951 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
952
953 });
954 });
955 });
956}
957
958B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
959{
960 BasicBlock* body = m_proc.addBlock();
961 BasicBlock* continuation = m_proc.addBlock();
962
963 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
964
965 m_currentBlock = body;
966 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
967
968 return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
969}
970
971B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
972{
973 return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
974}
975
976B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
977{
978 return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
979}
980
981auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
982{
983 // FIXME: This needs to do some kind of stack passing.
984
985 BasicBlock* taken = m_proc.addBlock();
986 BasicBlock* notTaken = m_proc.addBlock();
987 BasicBlock* continuation = m_proc.addBlock();
988
989 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
990 m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
991 taken->addPredecessor(m_currentBlock);
992 notTaken->addPredecessor(m_currentBlock);
993
994 m_currentBlock = taken;
995 result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
996 return { };
997}
998
999auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1000{
1001 unifyValuesWithBlock(currentStack, data.result);
1002 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1003 return addElseToUnreachable(data);
1004}
1005
1006auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1007{
1008 ASSERT(data.type() == BlockType::If);
1009 m_currentBlock = data.special;
1010 data.convertIfToBlock();
1011 return { };
1012}
1013
1014auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
1015{
1016 ASSERT(returnValues.size() <= 1);
1017 if (returnValues.size())
1018 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1019 else
1020 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1021 return { };
1022}
1023
1024auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1025{
1026 unifyValuesWithBlock(returnValues, data.resultForBranch());
1027
1028 BasicBlock* target = data.targetBlockForBranch();
1029 if (condition) {
1030 BasicBlock* continuation = m_proc.addBlock();
1031 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1032 m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1033 target->addPredecessor(m_currentBlock);
1034 continuation->addPredecessor(m_currentBlock);
1035 m_currentBlock = continuation;
1036 } else {
1037 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1038 target->addPredecessor(m_currentBlock);
1039 }
1040
1041 return { };
1042}
1043
1044auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1045{
1046 for (size_t i = 0; i < targets.size(); ++i)
1047 unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1048 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1049
1050 SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1051 switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1052 for (size_t i = 0; i < targets.size(); ++i)
1053 switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1054
1055 return { };
1056}
1057
1058auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1059{
1060 ControlData& data = entry.controlData;
1061
1062 unifyValuesWithBlock(expressionStack, data.result);
1063 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1064 data.continuation->addPredecessor(m_currentBlock);
1065
1066 return addEndToUnreachable(entry);
1067}
1068
1069
1070auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1071{
1072 ControlData& data = entry.controlData;
1073 m_currentBlock = data.continuation;
1074
1075 if (data.type() == BlockType::If) {
1076 data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1077 m_currentBlock->addPredecessor(data.special);
1078 }
1079
1080 for (Value* result : data.result) {
1081 m_currentBlock->append(result);
1082 entry.enclosedExpressionStack.append(result);
1083 }
1084
1085 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1086 if (data.type() == BlockType::TopLevel)
1087 return addReturn(entry.controlData, entry.enclosedExpressionStack);
1088
1089 return { };
1090}
1091
1092auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1093{
1094 ASSERT(signature.argumentCount() == args.size());
1095
1096 m_makesCalls = true;
1097
1098 Type returnType = signature.returnType();
1099 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1100
1101 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1102 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1103
1104 // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1105 Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1106 // The target instance is 0 unless the call is wasm->wasm.
1107 Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1108
1109 BasicBlock* isWasmBlock = m_proc.addBlock();
1110 BasicBlock* isEmbedderBlock = m_proc.addBlock();
1111 BasicBlock* continuation = m_proc.addBlock();
1112 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1113
1114 Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1115 [=] (PatchpointValue* patchpoint) {
1116 patchpoint->effects.writesPinned = true;
1117 patchpoint->effects.readsPinned = true;
1118 // We need to clobber all potential pinned registers since we might be leaving the instance.
1119 // We pessimistically assume we could be calling to something that is bounds checking.
1120 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1121 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1122 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1123 AllowMacroScratchRegisterUsage allowScratch(jit);
1124 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1125 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1126 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1127 });
1128 });
1129 });
1130 UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1131 isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1132
1133 // FIXME: Let's remove this indirection by creating a PIC friendly IC
1134 // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1135 // implement the IC to be over Context*.
1136 // https://bugs.webkit.org/show_bug.cgi?id=170375
1137 Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1138 Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
1139
1140 Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1141 [=] (PatchpointValue* patchpoint) {
1142 patchpoint->effects.writesPinned = true;
1143 patchpoint->effects.readsPinned = true;
1144 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1145 // We need to clobber all potential pinned registers since we might be leaving the instance.
1146 // We pessimistically assume we could be calling to something that is bounds checking.
1147 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1148 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1149 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1150 AllowMacroScratchRegisterUsage allowScratch(jit);
1151 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1152 });
1153 });
1154 UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1155 isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1156
1157 m_currentBlock = continuation;
1158
1159 if (returnType == Void)
1160 result = nullptr;
1161 else {
1162 result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1163 wasmCallResultUpsilon->setPhi(result);
1164 embedderCallResultUpsilon->setPhi(result);
1165 }
1166
1167 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1168 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
1169 } else {
1170 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1171 [=] (PatchpointValue* patchpoint) {
1172 patchpoint->effects.writesPinned = true;
1173 patchpoint->effects.readsPinned = true;
1174
1175 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1176 AllowMacroScratchRegisterUsage allowScratch(jit);
1177 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1178 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1179 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1180 });
1181 });
1182 });
1183 }
1184
1185 return { };
1186}
1187
1188auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1189{
1190 ExpressionType calleeIndex = args.takeLast();
1191 ASSERT(signature.argumentCount() == args.size());
1192
1193 m_makesCalls = true;
1194 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1195 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1196 // can be to the embedder for our stack check calculation.
1197 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1198
1199 ExpressionType callableFunctionBuffer;
1200 ExpressionType instancesBuffer;
1201 ExpressionType callableFunctionBufferLength;
1202 ExpressionType mask;
1203 {
1204 ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1205 instanceValue(), safeCast<int32_t>(Instance::offsetOfTable()));
1206 callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1207 table, safeCast<int32_t>(Table::offsetOfFunctions()));
1208 instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1209 table, safeCast<int32_t>(Table::offsetOfInstances()));
1210 callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1211 table, safeCast<int32_t>(Table::offsetOfLength()));
1212 mask = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(),
1213 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1214 table, safeCast<int32_t>(Table::offsetOfMask())));
1215 }
1216
1217 // Check the index we are looking for is valid.
1218 {
1219 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1220 m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
1221
1222 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1223 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1224 });
1225 }
1226
1227 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
1228
1229 if (Options::enableSpectreMitigations())
1230 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), mask, calleeIndex);
1231
1232 ExpressionType callableFunction;
1233 {
1234 // Compute the offset in the table index space we are looking for.
1235 ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1236 calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
1237 callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1238
1239 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1240 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1241 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1242 ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
1243 {
1244 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1245 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1246 calleeSignatureIndex,
1247 m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
1248
1249 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1250 this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1251 });
1252 }
1253
1254 // Check the signature matches the value we expect.
1255 {
1256 ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
1257 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1258 m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1259
1260 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1261 this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1262 });
1263 }
1264 }
1265
1266 // Do a context switch if needed.
1267 {
1268 Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1269 calleeIndex, constant(pointerType(), sizeof(Instance*)));
1270 Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1271 m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1272
1273 BasicBlock* continuation = m_proc.addBlock();
1274 BasicBlock* doContextSwitch = m_proc.addBlock();
1275
1276 Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1277 newContextInstance, instanceValue());
1278 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1279 isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1280
1281 PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1282 patchpoint->effects.writesPinned = true;
1283 // We pessimistically assume we're calling something with BoundsChecking memory.
1284 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1285 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1286 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1287 patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1288 patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1289 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
1290
1291 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1292 AllowMacroScratchRegisterUsage allowScratch(jit);
1293 GPRReg newContextInstance = params[0].gpr();
1294 GPRReg oldContextInstance = params[1].gpr();
1295 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1296 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1297 ASSERT(newContextInstance != baseMemory);
1298 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1299 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1300 jit.storeWasmContextInstance(newContextInstance);
1301 ASSERT(pinnedRegs.sizeRegister != baseMemory);
1302 // FIXME: We should support more than one memory size register
1303 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
1304 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1305 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
1306
1307 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1308 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1309
1310 jit.cageConditionally(Gigacage::Primitive, baseMemory, scratchOrSize);
1311 });
1312 doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1313
1314 m_currentBlock = continuation;
1315 }
1316
1317 ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1318 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1319 safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
1320
1321 Type returnType = signature.returnType();
1322 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1323 [=] (PatchpointValue* patchpoint) {
1324 patchpoint->effects.writesPinned = true;
1325 patchpoint->effects.readsPinned = true;
1326 // We need to clobber all potential pinned registers since we might be leaving the instance.
1327 // We pessimistically assume we're always calling something that is bounds checking so
1328 // because the wasm->wasm thunk unconditionally overrides the size registers.
1329 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1330 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1331 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1332
1333 patchpoint->append(calleeCode, ValueRep::SomeRegister);
1334 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1335 AllowMacroScratchRegisterUsage allowScratch(jit);
1336 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1337 });
1338 });
1339
1340 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1341 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
1342
1343 return { };
1344}
1345
1346void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1347{
1348 m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1349}
1350
1351void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1352{
1353 ASSERT(result.size() <= resultStack.size());
1354
1355 for (size_t i = 0; i < result.size(); ++i)
1356 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1357}
1358
1359static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1360{
1361 dataLog(comma, "ExpressionStack:");
1362 for (const auto& expression : expressionStack)
1363 dataLog(comma, *expression);
1364}
1365
1366void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1367{
1368 dataLogLn("Constants:");
1369 for (const auto& constant : m_constantPool)
1370 dataLogLn(deepDump(m_proc, constant.value));
1371
1372 dataLogLn("Processing Graph:");
1373 dataLog(m_proc);
1374 dataLogLn("With current block:", *m_currentBlock);
1375 dataLogLn("Control stack:");
1376 ASSERT(controlStack.size());
1377 for (size_t i = controlStack.size(); i--;) {
1378 dataLog(" ", controlStack[i].controlData, ": ");
1379 CommaPrinter comma(", ", "");
1380 dumpExpressionStack(comma, *expressionStack);
1381 expressionStack = &controlStack[i].enclosedExpressionStack;
1382 dataLogLn();
1383 }
1384 dataLogLn();
1385}
1386
1387auto B3IRGenerator::origin() -> Origin
1388{
1389 OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1390 ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1391 return bitwise_cast<Origin>(origin);
1392}
1393
1394Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1395{
1396 auto result = std::make_unique<InternalFunction>();
1397
1398 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1399 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1400
1401 Procedure procedure;
1402
1403 procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1404 if (origin.data())
1405 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1406 });
1407
1408 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1409 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1410 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1411 // optLevel=1.
1412 procedure.setNeedsUsedRegisters(false);
1413
1414 procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1415 ? Options::webAssemblyBBQOptimizationLevel()
1416 : Options::webAssemblyOMGOptimizationLevel());
1417
1418 B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1419 FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1420 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1421
1422 irGenerator.insertConstants();
1423
1424 procedure.resetReachability();
1425 if (!ASSERT_DISABLED)
1426 validate(procedure, "After parsing:\n");
1427
1428 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1429 fixSSA(procedure);
1430 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1431
1432 {
1433 B3::prepareForGeneration(procedure);
1434 B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1435 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1436 result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1437 }
1438
1439 return result;
1440}
1441
1442// Custom wasm ops. These are the ones too messy to do in wasm.json.
1443
1444void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1445{
1446 ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1447 const B3::Type type = left->type();
1448
1449 {
1450 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1451 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1452
1453 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1454 this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1455 });
1456 }
1457
1458 if (operation == Div) {
1459 int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1460
1461 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1462 m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1463 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1464 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1465
1466 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1467 this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1468 });
1469 }
1470}
1471
1472template<>
1473auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1474{
1475 const B3::Opcode op = Div;
1476 emitChecksForModOrDiv(op, left, right);
1477 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1478 return { };
1479}
1480
1481template<>
1482auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1483{
1484 const B3::Opcode op = Mod;
1485 emitChecksForModOrDiv(op, left, right);
1486 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1487 return { };
1488}
1489
1490template<>
1491auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1492{
1493 const B3::Opcode op = UDiv;
1494 emitChecksForModOrDiv(op, left, right);
1495 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1496 return { };
1497}
1498
1499template<>
1500auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1501{
1502 const B3::Opcode op = UMod;
1503 emitChecksForModOrDiv(op, left, right);
1504 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1505 return { };
1506}
1507
1508template<>
1509auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1510{
1511 const B3::Opcode op = Div;
1512 emitChecksForModOrDiv(op, left, right);
1513 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1514 return { };
1515}
1516
1517template<>
1518auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1519{
1520 const B3::Opcode op = Mod;
1521 emitChecksForModOrDiv(op, left, right);
1522 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1523 return { };
1524}
1525
1526template<>
1527auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1528{
1529 const B3::Opcode op = UDiv;
1530 emitChecksForModOrDiv(op, left, right);
1531 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1532 return { };
1533}
1534
1535template<>
1536auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1537{
1538 const B3::Opcode op = UMod;
1539 emitChecksForModOrDiv(op, left, right);
1540 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1541 return { };
1542}
1543
1544template<>
1545auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1546{
1547 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1548 patchpoint->append(arg, ValueRep::SomeRegister);
1549 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1550 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1551 });
1552 patchpoint->effects = Effects::none();
1553 result = patchpoint;
1554 return { };
1555}
1556
1557template<>
1558auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1559{
1560 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1561 patchpoint->append(arg, ValueRep::SomeRegister);
1562 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1563 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1564 });
1565 patchpoint->effects = Effects::none();
1566 result = patchpoint;
1567 return { };
1568}
1569
1570template<>
1571auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1572{
1573#if CPU(X86_64)
1574 if (MacroAssembler::supportsCountPopulation()) {
1575 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1576 patchpoint->append(arg, ValueRep::SomeRegister);
1577 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1578 jit.countPopulation32(params[1].gpr(), params[0].gpr());
1579 });
1580 patchpoint->effects = Effects::none();
1581 result = patchpoint;
1582 return { };
1583 }
1584#endif
1585
1586 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1587 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1588 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1589 return { };
1590}
1591
1592template<>
1593auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1594{
1595#if CPU(X86_64)
1596 if (MacroAssembler::supportsCountPopulation()) {
1597 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1598 patchpoint->append(arg, ValueRep::SomeRegister);
1599 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1600 jit.countPopulation64(params[1].gpr(), params[0].gpr());
1601 });
1602 patchpoint->effects = Effects::none();
1603 result = patchpoint;
1604 return { };
1605 }
1606#endif
1607
1608 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1609 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1610 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1611 return { };
1612}
1613
1614template<>
1615auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1616{
1617 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1618 if (isX86())
1619 patchpoint->numGPScratchRegisters = 1;
1620 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1621 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1622 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1623 AllowMacroScratchRegisterUsage allowScratch(jit);
1624#if CPU(X86_64)
1625 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1626#else
1627 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1628#endif
1629 });
1630 patchpoint->effects = Effects::none();
1631 result = patchpoint;
1632 return { };
1633}
1634
1635template<>
1636auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1637{
1638 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1639 if (isX86())
1640 patchpoint->numGPScratchRegisters = 1;
1641 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1642 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1643 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1644 AllowMacroScratchRegisterUsage allowScratch(jit);
1645#if CPU(X86_64)
1646 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1647#else
1648 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1649#endif
1650 });
1651 patchpoint->effects = Effects::none();
1652 result = patchpoint;
1653 return { };
1654}
1655
1656template<>
1657auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1658{
1659 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1660 patchpoint->append(arg, ValueRep::SomeRegister);
1661 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1662 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1663 });
1664 patchpoint->effects = Effects::none();
1665 result = patchpoint;
1666 return { };
1667}
1668
1669template<>
1670auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1671{
1672 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1673 patchpoint->append(arg, ValueRep::SomeRegister);
1674 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1675 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1676 });
1677 patchpoint->effects = Effects::none();
1678 result = patchpoint;
1679 return { };
1680}
1681
1682template<>
1683auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1684{
1685 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1686 patchpoint->append(arg, ValueRep::SomeRegister);
1687 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1688 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1689 });
1690 patchpoint->effects = Effects::none();
1691 result = patchpoint;
1692 return { };
1693}
1694
1695template<>
1696auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1697{
1698 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1699 patchpoint->append(arg, ValueRep::SomeRegister);
1700 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1701 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1702 });
1703 patchpoint->effects = Effects::none();
1704 result = patchpoint;
1705 return { };
1706}
1707
1708template<>
1709auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1710{
1711 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1712 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1713 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1714 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1715 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1716 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1717 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1718 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1719 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1720 });
1721 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1722 patchpoint->append(arg, ValueRep::SomeRegister);
1723 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1724 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1725 });
1726 patchpoint->effects = Effects::none();
1727 result = patchpoint;
1728 return { };
1729}
1730
1731template<>
1732auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1733{
1734 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1735 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1736 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1737 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1738 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1739 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1740 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1741 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1742 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1743 });
1744 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1745 patchpoint->append(arg, ValueRep::SomeRegister);
1746 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1747 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1748 });
1749 patchpoint->effects = Effects::none();
1750 result = patchpoint;
1751 return { };
1752}
1753
1754
1755template<>
1756auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1757{
1758 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1759 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1760 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1761 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1762 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1763 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1764 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1765 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1766 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1767 });
1768 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1769 patchpoint->append(arg, ValueRep::SomeRegister);
1770 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1771 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1772 });
1773 patchpoint->effects = Effects::none();
1774 result = patchpoint;
1775 return { };
1776}
1777
1778template<>
1779auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1780{
1781 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1782 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1783 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1784 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1785 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1786 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1787 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1788 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1789 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1790 });
1791 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1792 patchpoint->append(arg, ValueRep::SomeRegister);
1793 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1794 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1795 });
1796 patchpoint->effects = Effects::none();
1797 result = patchpoint;
1798 return { };
1799}
1800
1801template<>
1802auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1803{
1804 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1805 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1806 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1807 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1808 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1809 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1810 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1811 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1812 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1813 });
1814 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1815 patchpoint->append(arg, ValueRep::SomeRegister);
1816 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1817 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1818 });
1819 patchpoint->effects = Effects::none();
1820 result = patchpoint;
1821 return { };
1822}
1823
1824template<>
1825auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1826{
1827 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1828 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1829 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1830 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1831 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1832 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1833 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1834 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1835 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1836 });
1837
1838 Value* signBitConstant;
1839 if (isX86()) {
1840 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1841 // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1842 // so we can pool them if needed.
1843 signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1844 }
1845 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1846 patchpoint->append(arg, ValueRep::SomeRegister);
1847 if (isX86()) {
1848 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1849 patchpoint->numFPScratchRegisters = 1;
1850 }
1851 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1852 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1853 AllowMacroScratchRegisterUsage allowScratch(jit);
1854 FPRReg scratch = InvalidFPRReg;
1855 FPRReg constant = InvalidFPRReg;
1856 if (isX86()) {
1857 scratch = params.fpScratch(0);
1858 constant = params[2].fpr();
1859 }
1860 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1861 });
1862 patchpoint->effects = Effects::none();
1863 result = patchpoint;
1864 return { };
1865}
1866
1867template<>
1868auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1869{
1870 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1871 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1872 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1873 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1874 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1875 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1876 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1877 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1878 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1879 });
1880 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1881 patchpoint->append(arg, ValueRep::SomeRegister);
1882 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1883 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1884 });
1885 patchpoint->effects = Effects::none();
1886 result = patchpoint;
1887 return { };
1888}
1889
1890template<>
1891auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1892{
1893 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1894 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1895 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1896 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1897 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1898 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1899 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1900 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1901 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1902 });
1903
1904 Value* signBitConstant;
1905 if (isX86()) {
1906 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1907 // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1908 // so we can pool them if needed.
1909 signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1910 }
1911 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1912 patchpoint->append(arg, ValueRep::SomeRegister);
1913 if (isX86()) {
1914 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1915 patchpoint->numFPScratchRegisters = 1;
1916 }
1917 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1918 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1919 AllowMacroScratchRegisterUsage allowScratch(jit);
1920 FPRReg scratch = InvalidFPRReg;
1921 FPRReg constant = InvalidFPRReg;
1922 if (isX86()) {
1923 scratch = params.fpScratch(0);
1924 constant = params[2].fpr();
1925 }
1926 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1927 });
1928 patchpoint->effects = Effects::none();
1929 result = patchpoint;
1930 return { };
1931}
1932
1933} } // namespace JSC::Wasm
1934
1935#include "WasmB3IRGeneratorInlines.h"
1936
1937#endif // ENABLE(WEBASSEMBLY)
1938