1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmAirIRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AirCode.h"
32#include "AirGenerate.h"
33#include "AirOpcodeUtils.h"
34#include "AirValidate.h"
35#include "AllowMacroScratchRegisterUsageIf.h"
36#include "B3CCallValue.h"
37#include "B3CheckSpecial.h"
38#include "B3CheckValue.h"
39#include "B3PatchpointSpecial.h"
40#include "B3Procedure.h"
41#include "B3ProcedureInlines.h"
42#include "BinarySwitch.h"
43#include "DisallowMacroScratchRegisterUsage.h"
44#include "JSCInlines.h"
45#include "ScratchRegisterAllocator.h"
46#include "VirtualRegister.h"
47#include "WasmCallingConvention.h"
48#include "WasmContextInlines.h"
49#include "WasmExceptionType.h"
50#include "WasmFunctionParser.h"
51#include "WasmInstance.h"
52#include "WasmMemory.h"
53#include "WasmOMGPlan.h"
54#include "WasmOpcodeOrigin.h"
55#include "WasmSignatureInlines.h"
56#include "WasmThunks.h"
57#include <limits>
58#include <wtf/Box.h>
59#include <wtf/Optional.h>
60#include <wtf/StdLibExtras.h>
61
62namespace JSC { namespace Wasm {
63
64using namespace B3::Air;
65
66struct ConstrainedTmp {
67 ConstrainedTmp(Tmp tmp)
68 : ConstrainedTmp(tmp, tmp.isReg() ? B3::ValueRep::reg(tmp.reg()) : B3::ValueRep::SomeRegister)
69 { }
70
71 ConstrainedTmp(Tmp tmp, B3::ValueRep rep)
72 : tmp(tmp)
73 , rep(rep)
74 {
75 }
76
77 Tmp tmp;
78 B3::ValueRep rep;
79};
80
81class TypedTmp {
82public:
83 constexpr TypedTmp()
84 : m_tmp()
85 , m_type(Type::Void)
86 { }
87
88 TypedTmp(Tmp tmp, Type type)
89 : m_tmp(tmp)
90 , m_type(type)
91 { }
92
93 TypedTmp(const TypedTmp&) = default;
94 TypedTmp(TypedTmp&&) = default;
95 TypedTmp& operator=(TypedTmp&&) = default;
96 TypedTmp& operator=(const TypedTmp&) = default;
97
98 bool operator==(const TypedTmp& other) const
99 {
100 return m_tmp == other.m_tmp && m_type == other.m_type;
101 }
102 bool operator!=(const TypedTmp& other) const
103 {
104 return !(*this == other);
105 }
106
107 explicit operator bool() const { return !!tmp(); }
108
109 operator Tmp() const { return tmp(); }
110 operator Arg() const { return Arg(tmp()); }
111 Tmp tmp() const { return m_tmp; }
112 Type type() const { return m_type; }
113
114private:
115
116 Tmp m_tmp;
117 Type m_type;
118};
119
120class AirIRGenerator {
121public:
122 struct ControlData {
123 ControlData(B3::Origin origin, Type returnType, TypedTmp resultTmp, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
124 : blockType(type)
125 , continuation(continuation)
126 , special(special)
127 , returnType(returnType)
128 {
129 UNUSED_PARAM(origin); // FIXME: Use origin.
130 if (resultTmp) {
131 ASSERT(returnType != Type::Void);
132 result.append(resultTmp);
133 } else
134 ASSERT(returnType == Type::Void);
135 }
136
137 ControlData()
138 {
139 }
140
141 void dump(PrintStream& out) const
142 {
143 switch (type()) {
144 case BlockType::If:
145 out.print("If: ");
146 break;
147 case BlockType::Block:
148 out.print("Block: ");
149 break;
150 case BlockType::Loop:
151 out.print("Loop: ");
152 break;
153 case BlockType::TopLevel:
154 out.print("TopLevel: ");
155 break;
156 }
157 out.print("Continuation: ", *continuation, ", Special: ");
158 if (special)
159 out.print(*special);
160 else
161 out.print("None");
162 }
163
164 BlockType type() const { return blockType; }
165
166 Type signature() const { return returnType; }
167
168 bool hasNonVoidSignature() const { return result.size(); }
169
170 BasicBlock* targetBlockForBranch()
171 {
172 if (type() == BlockType::Loop)
173 return special;
174 return continuation;
175 }
176
177 void convertIfToBlock()
178 {
179 ASSERT(type() == BlockType::If);
180 blockType = BlockType::Block;
181 special = nullptr;
182 }
183
184 using ResultList = Vector<TypedTmp, 1>;
185
186 ResultList resultForBranch() const
187 {
188 if (type() == BlockType::Loop)
189 return ResultList();
190 return result;
191 }
192
193 private:
194 friend class AirIRGenerator;
195 BlockType blockType;
196 BasicBlock* continuation;
197 BasicBlock* special;
198 ResultList result;
199 Type returnType;
200 };
201
202 using ExpressionType = TypedTmp;
203 using ControlType = ControlData;
204 using ExpressionList = Vector<ExpressionType, 1>;
205 using ResultList = ControlData::ResultList;
206 using ControlEntry = FunctionParser<AirIRGenerator>::ControlEntry;
207
208 static ExpressionType emptyExpression() { return { }; };
209
210 using ErrorType = String;
211 using UnexpectedResult = Unexpected<ErrorType>;
212 using Result = Expected<std::unique_ptr<InternalFunction>, ErrorType>;
213 using PartialResult = Expected<void, ErrorType>;
214
215 template <typename ...Args>
216 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
217 {
218 using namespace FailureHelper; // See ADL comment in WasmParser.h.
219 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
220 }
221
222#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
223 if (UNLIKELY(condition)) \
224 return fail(__VA_ARGS__); \
225 } while (0)
226
227 AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, ThrowWasmException, const Signature&);
228
229 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
230 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
231 ExpressionType addConstant(Type, uint64_t);
232 ExpressionType addConstant(BasicBlock*, Type, uint64_t);
233
234 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
235
236 // Locals
237 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
238 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
239
240 // Globals
241 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
242 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
243
244 // Memory
245 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
246 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
247 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
248 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
249
250 // Basic operators
251 template<OpType>
252 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
253 template<OpType>
254 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
255 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
256
257 // Control flow
258 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
259 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
260 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
261 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
262 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
263 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
264
265 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
266 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
267 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
268 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
269 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
270
271 // Calls
272 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
273 PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
274 PartialResult WARN_UNUSED_RETURN addUnreachable();
275
276 PartialResult addShift(Type, B3::Air::Opcode, ExpressionType value, ExpressionType shift, ExpressionType& result);
277 PartialResult addIntegerSub(B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
278 PartialResult addFloatingPointAbs(B3::Air::Opcode, ExpressionType value, ExpressionType& result);
279 PartialResult addFloatingPointBinOp(Type, B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
280
281 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
282 void setParser(FunctionParser<AirIRGenerator>* parser) { m_parser = parser; };
283
284 static Vector<Tmp> toTmpVector(const Vector<TypedTmp>& vector)
285 {
286 Vector<Tmp> result;
287 for (const auto& item : vector)
288 result.append(item.tmp());
289 return result;
290 }
291
292 ALWAYS_INLINE void didKill(const ExpressionType& typedTmp)
293 {
294 Tmp tmp = typedTmp.tmp();
295 if (!tmp)
296 return;
297 if (tmp.isGP())
298 m_freeGPs.append(tmp);
299 else
300 m_freeFPs.append(tmp);
301 }
302
303private:
304 ALWAYS_INLINE void validateInst(Inst& inst)
305 {
306 if (!ASSERT_DISABLED) {
307 if (!inst.isValidForm()) {
308 dataLogLn(inst);
309 CRASH();
310 }
311 }
312 }
313
314 static Arg extractArg(const TypedTmp& tmp) { return tmp.tmp(); }
315 static Arg extractArg(const Tmp& tmp) { return Arg(tmp); }
316 static Arg extractArg(const Arg& arg) { return arg; }
317
318 template<typename... Arguments>
319 void append(BasicBlock* block, Kind kind, Arguments&&... arguments)
320 {
321 // FIXME: Find a way to use origin here.
322 auto& inst = block->append(kind, nullptr, extractArg(arguments)...);
323 validateInst(inst);
324 }
325
326 template<typename... Arguments>
327 void append(Kind kind, Arguments&&... arguments)
328 {
329 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
330 }
331
332 template<typename... Arguments>
333 void appendEffectful(B3::Air::Opcode op, Arguments&&... arguments)
334 {
335 Kind kind = op;
336 kind.effects = true;
337 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
338 }
339
340 Tmp newTmp(B3::Bank bank)
341 {
342 switch (bank) {
343 case B3::GP:
344 if (m_freeGPs.size())
345 return m_freeGPs.takeLast();
346 break;
347 case B3::FP:
348 if (m_freeFPs.size())
349 return m_freeFPs.takeLast();
350 break;
351 }
352 return m_code.newTmp(bank);
353 }
354
355 TypedTmp g32() { return { newTmp(B3::GP), Type::I32 }; }
356 TypedTmp g64() { return { newTmp(B3::GP), Type::I64 }; }
357 TypedTmp f32() { return { newTmp(B3::FP), Type::F32 }; }
358 TypedTmp f64() { return { newTmp(B3::FP), Type::F64 }; }
359
360 TypedTmp tmpForType(Type type)
361 {
362 switch (type) {
363 case Type::I32:
364 return g32();
365 case Type::I64:
366 case Type::Anyref:
367 return g64();
368 case Type::F32:
369 return f32();
370 case Type::F64:
371 return f64();
372 case Type::Void:
373 return { };
374 default:
375 RELEASE_ASSERT_NOT_REACHED();
376 }
377 }
378
379 B3::PatchpointValue* addPatchpoint(B3::Type type)
380 {
381 return m_proc.add<B3::PatchpointValue>(type, B3::Origin());
382 }
383
384 template <typename ...Args>
385 void emitPatchpoint(B3::PatchpointValue* patch, Tmp result, Args... theArgs)
386 {
387 emitPatchpoint(m_currentBlock, patch, result, std::forward<Args>(theArgs)...);
388 }
389
390 template <typename ...Args>
391 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Args... theArgs)
392 {
393 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp, sizeof...(Args)>::from(theArgs...));
394 }
395
396 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result)
397 {
398 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp>());
399 }
400
401 template <size_t inlineSize>
402 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Vector<ConstrainedTmp, inlineSize>&& args)
403 {
404 if (!m_patchpointSpecial)
405 m_patchpointSpecial = static_cast<B3::PatchpointSpecial*>(m_code.addSpecial(std::make_unique<B3::PatchpointSpecial>()));
406
407 Inst inst(Patch, patch, Arg::special(m_patchpointSpecial));
408 Inst resultMov;
409 if (result) {
410 ASSERT(patch->type() != B3::Void);
411 switch (patch->resultConstraint.kind()) {
412 case B3::ValueRep::Register:
413 inst.args.append(Tmp(patch->resultConstraint.reg()));
414 resultMov = Inst(result.isGP() ? Move : MoveDouble, nullptr, Tmp(patch->resultConstraint.reg()), result);
415 break;
416 case B3::ValueRep::SomeRegister:
417 inst.args.append(result);
418 break;
419 default:
420 RELEASE_ASSERT_NOT_REACHED();
421 }
422 } else
423 ASSERT(patch->type() == B3::Void);
424
425 for (ConstrainedTmp& tmp : args) {
426 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
427 // validation. We should abstrcat Patch enough so ValueRep's don't need to be
428 // backed by Values.
429 // https://bugs.webkit.org/show_bug.cgi?id=194040
430 B3::Value* dummyValue = m_proc.addConstant(B3::Origin(), tmp.tmp.isGP() ? B3::Int64 : B3::Double, 0);
431 patch->append(dummyValue, tmp.rep);
432 switch (tmp.rep.kind()) {
433 case B3::ValueRep::SomeRegister:
434 inst.args.append(tmp.tmp);
435 break;
436 case B3::ValueRep::Register:
437 patch->earlyClobbered().clear(tmp.rep.reg());
438 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, tmp.rep.reg());
439 inst.args.append(Tmp(tmp.rep.reg()));
440 break;
441 case B3::ValueRep::StackArgument: {
442 auto arg = Arg::callArg(tmp.rep.offsetFromSP());
443 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, arg);
444 inst.args.append(arg);
445 break;
446 }
447 default:
448 RELEASE_ASSERT_NOT_REACHED();
449 }
450 }
451
452 if (patch->resultConstraint.isReg())
453 patch->lateClobbered().clear(patch->resultConstraint.reg());
454 for (unsigned i = patch->numGPScratchRegisters; i--;)
455 inst.args.append(g64().tmp());
456 for (unsigned i = patch->numFPScratchRegisters; i--;)
457 inst.args.append(f64().tmp());
458
459 validateInst(inst);
460 basicBlock->append(WTFMove(inst));
461 if (resultMov) {
462 validateInst(resultMov);
463 basicBlock->append(WTFMove(resultMov));
464 }
465 }
466
467 template <typename Branch, typename Generator>
468 void emitCheck(const Branch& makeBranch, const Generator& generator)
469 {
470 // We fail along the truthy edge of 'branch'.
471 Inst branch = makeBranch();
472
473 // FIXME: Make a hashmap of these.
474 B3::CheckSpecial::Key key(branch);
475 B3::CheckSpecial* special = static_cast<B3::CheckSpecial*>(m_code.addSpecial(std::make_unique<B3::CheckSpecial>(key)));
476
477 // FIXME: Remove the need for dummy values
478 // https://bugs.webkit.org/show_bug.cgi?id=194040
479 B3::Value* dummyPredicate = m_proc.addConstant(B3::Origin(), B3::Int32, 42);
480 B3::CheckValue* checkValue = m_proc.add<B3::CheckValue>(B3::Check, B3::Origin(), dummyPredicate);
481 checkValue->setGenerator(generator);
482
483 Inst inst(Patch, checkValue, Arg::special(special));
484 inst.args.appendVector(branch.args);
485 m_currentBlock->append(WTFMove(inst));
486 }
487
488 template <typename Func, typename ...Args>
489 void emitCCall(Func func, TypedTmp result, Args... args)
490 {
491 emitCCall(m_currentBlock, func, result, std::forward<Args>(args)...);
492 }
493 template <typename Func, typename ...Args>
494 void emitCCall(BasicBlock* block, Func func, TypedTmp result, Args... theArgs)
495 {
496 B3::Type resultType = B3::Void;
497 if (result) {
498 switch (result.type()) {
499 case Type::I32:
500 resultType = B3::Int32;
501 break;
502 case Type::I64:
503 resultType = B3::Int64;
504 break;
505 case Type::F32:
506 resultType = B3::Float;
507 break;
508 case Type::F64:
509 resultType = B3::Double;
510 break;
511 default:
512 RELEASE_ASSERT_NOT_REACHED();
513 }
514 }
515
516 auto makeDummyValue = [&] (Tmp tmp) {
517 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
518 // validation. We should abstrcat CCall enough so we're not reliant on arguments
519 // to the B3::CCallValue.
520 // https://bugs.webkit.org/show_bug.cgi?id=194040
521 if (tmp.isGP())
522 return m_proc.addConstant(B3::Origin(), B3::Int64, 0);
523 return m_proc.addConstant(B3::Origin(), B3::Double, 0);
524 };
525
526 B3::Value* dummyFunc = m_proc.addConstant(B3::Origin(), B3::Int64, bitwise_cast<uintptr_t>(func));
527 B3::Value* origin = m_proc.add<B3::CCallValue>(resultType, B3::Origin(), B3::Effects::none(), dummyFunc, makeDummyValue(theArgs)...);
528
529 Inst inst(CCall, origin);
530
531 Tmp callee = g64();
532 append(Move, Arg::immPtr(tagCFunctionPtr<void*>(func, B3CCallPtrTag)), callee);
533 inst.args.append(callee);
534
535 if (result)
536 inst.args.append(result.tmp());
537
538 for (Tmp tmp : Vector<Tmp, sizeof...(Args)>::from(theArgs.tmp()...))
539 inst.args.append(tmp);
540
541 block->append(WTFMove(inst));
542 }
543
544 static B3::Air::Opcode moveOpForValueType(Type type)
545 {
546 switch (type) {
547 case Type::I32:
548 return Move32;
549 case Type::I64:
550 return Move;
551 case Type::F32:
552 return MoveFloat;
553 case Type::F64:
554 return MoveDouble;
555 default:
556 RELEASE_ASSERT_NOT_REACHED();
557 }
558 }
559
560 void emitThrowException(CCallHelpers&, ExceptionType);
561
562 void emitTierUpCheck(uint32_t decrementCount, B3::Origin);
563
564 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
565 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
566 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
567
568 void unify(const ExpressionType& dst, const ExpressionType& source);
569 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
570
571 template <typename IntType>
572 void emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right);
573
574 template <typename IntType>
575 void emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
576
577 enum class MinOrMax { Min, Max };
578
579 PartialResult addFloatingPointMinOrMax(Type, MinOrMax, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
580
581 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
582
583 void restoreWasmContextInstance(BasicBlock*, TypedTmp);
584 enum class RestoreCachedStackLimit { No, Yes };
585 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, TypedTmp instance, BasicBlock*);
586
587 B3::Origin origin();
588
589 FunctionParser<AirIRGenerator>* m_parser { nullptr };
590 const ModuleInformation& m_info;
591 const MemoryMode m_mode { MemoryMode::BoundsChecking };
592 const unsigned m_functionIndex { UINT_MAX };
593 const TierUpCount* m_tierUp { nullptr };
594
595 B3::Procedure& m_proc;
596 Code& m_code;
597 BasicBlock* m_currentBlock { nullptr };
598 BasicBlock* m_rootBlock { nullptr };
599 Vector<TypedTmp> m_locals;
600 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
601 GPRReg m_memoryBaseGPR { InvalidGPRReg };
602 GPRReg m_memorySizeGPR { InvalidGPRReg };
603 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
604 bool m_makesCalls { false };
605
606 Vector<Tmp, 8> m_freeGPs;
607 Vector<Tmp, 8> m_freeFPs;
608
609 TypedTmp m_instanceValue; // Always use the accessor below to ensure the instance value is materialized when used.
610 bool m_usesInstanceValue { false };
611 TypedTmp instanceValue()
612 {
613 m_usesInstanceValue = true;
614 return m_instanceValue;
615 }
616
617 uint32_t m_maxNumJSCallArguments { 0 };
618
619 B3::PatchpointSpecial* m_patchpointSpecial { nullptr };
620};
621
622// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
623int32_t AirIRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
624{
625 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
626 auto previousPtr = ptr;
627 ptr = g64();
628 auto constant = g64();
629 append(Move, Arg::bigImm(offset), constant);
630 append(Add64, constant, previousPtr, ptr);
631 return 0;
632 }
633 return offset;
634}
635
636void AirIRGenerator::restoreWasmContextInstance(BasicBlock* block, TypedTmp instance)
637{
638 if (Context::useFastTLS()) {
639 auto* patchpoint = addPatchpoint(B3::Void);
640 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
641 patchpoint->clobber(RegisterSet::macroScratchRegisters());
642 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
643 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
644 jit.storeWasmContextInstance(params[0].gpr());
645 });
646 emitPatchpoint(block, patchpoint, Tmp(), instance);
647 return;
648 }
649
650 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
651 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
652 auto* patchpoint = addPatchpoint(B3::Void);
653 B3::Effects effects = B3::Effects::none();
654 effects.writesPinned = true;
655 effects.reads = B3::HeapRange::top();
656 patchpoint->effects = effects;
657 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
658 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
659 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& param) {
660 jit.move(param[0].gpr(), wasmContextInstanceGPR);
661 });
662 emitPatchpoint(block, patchpoint, Tmp(), instance);
663}
664
665AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException, const Signature& signature)
666 : m_info(info)
667 , m_mode(mode)
668 , m_functionIndex(functionIndex)
669 , m_tierUp(tierUp)
670 , m_proc(procedure)
671 , m_code(m_proc.code())
672 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
673{
674 m_currentBlock = m_code.addBlock();
675 m_rootBlock = m_currentBlock;
676
677 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
678 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
679
680 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
681 m_code.pinRegister(m_memoryBaseGPR);
682
683 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
684 if (!Context::useFastTLS())
685 m_code.pinRegister(m_wasmContextInstanceGPR);
686
687 if (mode != MemoryMode::Signaling) {
688 m_memorySizeGPR = pinnedRegs.sizeRegister;
689 m_code.pinRegister(m_memorySizeGPR);
690 }
691
692 if (throwWasmException)
693 Thunks::singleton().setThrowWasmException(throwWasmException);
694
695 if (info.memory) {
696 switch (m_mode) {
697 case MemoryMode::BoundsChecking:
698 break;
699 case MemoryMode::Signaling:
700 // Most memory accesses in signaling mode don't do an explicit
701 // exception check because they can rely on fault handling to detect
702 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
703 // thunk to exist so that it can jump to that thunk.
704 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
705 CRASH();
706 break;
707 }
708 }
709
710 m_code.setNumEntrypoints(1);
711
712 GPRReg contextInstance = Context::useFastTLS() ? wasmCallingConventionAir().prologueScratch(1) : m_wasmContextInstanceGPR;
713
714 Ref<B3::Air::PrologueGenerator> prologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([=] (CCallHelpers& jit, B3::Air::Code& code) {
715 AllowMacroScratchRegisterUsage allowScratch(jit);
716 code.emitDefaultPrologue(jit);
717
718 {
719 GPRReg calleeGPR = wasmCallingConventionAir().prologueScratch(0);
720 auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
721 jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {
722 compilation->calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
723 });
724 jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);
725 jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
726 }
727
728 {
729 const Checked<int32_t> wasmFrameSize = m_code.frameSize();
730 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
731 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
732 // This allows us to elide stack checks for functions that are terminal nodes in the call
733 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
734 // having any such terminal node have its parent caller include some extra size in its
735 // own check for it. The goal here is twofold:
736 // 1. Emit less code.
737 // 2. Try to speed things up by skipping stack checks.
738 minimumParentCheckSize,
739 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
740 // spill all arguments to the stack, we ensure that a stack check here covers the
741 // stack that such a stub would use.
742 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
743 ));
744 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
745 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
746 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
747
748 // This allows leaf functions to not do stack checks if their frame size is within
749 // certain limits since their caller would have already done the check.
750 if (needsOverflowCheck) {
751 GPRReg scratch = wasmCallingConventionAir().prologueScratch(0);
752
753 if (Context::useFastTLS())
754 jit.loadWasmContextInstance(contextInstance);
755
756 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), GPRInfo::callFrameRegister, scratch);
757 MacroAssembler::JumpList overflow;
758 if (UNLIKELY(needUnderflowCheck))
759 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch, GPRInfo::callFrameRegister));
760 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch, CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit())));
761 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
762 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
763 });
764 } else if (m_usesInstanceValue && Context::useFastTLS()) {
765 // No overflow check is needed, but the instance values still needs to be correct.
766 jit.loadWasmContextInstance(contextInstance);
767 }
768 }
769 });
770
771 m_code.setPrologueForEntrypoint(0, WTFMove(prologueGenerator));
772
773 if (Context::useFastTLS()) {
774 m_instanceValue = g64();
775 // FIXME: Would be nice to only do this if we use instance value.
776 append(Move, Tmp(contextInstance), m_instanceValue);
777 } else
778 m_instanceValue = { Tmp(contextInstance), Type::I64 };
779
780 ASSERT(!m_locals.size());
781 m_locals.grow(signature.argumentCount());
782 for (unsigned i = 0; i < signature.argumentCount(); ++i) {
783 Type type = signature.argument(i);
784 m_locals[i] = tmpForType(type);
785 }
786
787 wasmCallingConventionAir().loadArguments(signature, [&] (const Arg& arg, unsigned i) {
788 switch (signature.argument(i)) {
789 case Type::I32:
790 append(Move32, arg, m_locals[i]);
791 break;
792 case Type::I64:
793 case Type::Anyref:
794 append(Move, arg, m_locals[i]);
795 break;
796 case Type::F32:
797 append(MoveFloat, arg, m_locals[i]);
798 break;
799 case Type::F64:
800 append(MoveDouble, arg, m_locals[i]);
801 break;
802 default:
803 RELEASE_ASSERT_NOT_REACHED();
804 }
805 });
806
807 emitTierUpCheck(TierUpCount::functionEntryDecrement(), B3::Origin());
808}
809
810void AirIRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, TypedTmp instance, BasicBlock* block)
811{
812 restoreWasmContextInstance(block, instance);
813
814 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
815 // The Instance caches the stack limit, but also knows where its canonical location is.
816 static_assert(sizeof(decltype(static_cast<Instance*>(nullptr)->cachedStackLimit())) == sizeof(uint64_t), "");
817
818 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfPointerToActualStackLimit(), B3::Width64));
819 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedStackLimit(), B3::Width64));
820 auto temp = g64();
821 append(block, Move, Arg::addr(instanceValue(), Instance::offsetOfPointerToActualStackLimit()), temp);
822 append(block, Move, Arg::addr(temp), temp);
823 append(block, Move, temp, Arg::addr(instanceValue(), Instance::offsetOfCachedStackLimit()));
824 }
825
826 if (!!memory) {
827 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
828 RegisterSet clobbers;
829 clobbers.set(pinnedRegs->baseMemoryPointer);
830 clobbers.set(pinnedRegs->sizeRegister);
831 if (!isARM64())
832 clobbers.set(RegisterSet::macroScratchRegisters());
833
834 auto* patchpoint = addPatchpoint(B3::Void);
835 B3::Effects effects = B3::Effects::none();
836 effects.writesPinned = true;
837 effects.reads = B3::HeapRange::top();
838 patchpoint->effects = effects;
839 patchpoint->clobber(clobbers);
840 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
841
842 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
843 RELEASE_ASSERT(!Gigacage::isEnabled(Gigacage::Primitive) || !isARM64());
844 AllowMacroScratchRegisterUsageIf allowScratch(jit, !isARM64());
845 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
846 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
847
848 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
849 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
850
851 jit.cageConditionally(Gigacage::Primitive, baseMemory, scratchOrSize);
852 });
853
854 emitPatchpoint(block, patchpoint, Tmp(), instance);
855 }
856}
857
858void AirIRGenerator::emitThrowException(CCallHelpers& jit, ExceptionType type)
859{
860 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
861 auto jumpToExceptionStub = jit.jump();
862
863 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
864 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
865 });
866}
867
868auto AirIRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
869{
870 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
871 totalBytesChecked += m_locals.size();
872 uint32_t totalBytes;
873 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
874
875 for (uint32_t i = 0; i < count; ++i) {
876 auto local = tmpForType(type);
877 m_locals.uncheckedAppend(local);
878 switch (type) {
879 case Type::I32:
880 case Type::I64: {
881 append(Xor64, local, local);
882 break;
883 }
884 case Type::F32:
885 case Type::F64: {
886 auto temp = g64();
887 // IEEE 754 "0" is just int32/64 zero.
888 append(Xor64, temp, temp);
889 append(type == Type::F32 ? Move32ToFloat : Move64ToDouble, temp, local);
890 break;
891 }
892 default:
893 RELEASE_ASSERT_NOT_REACHED();
894 }
895 }
896 return { };
897}
898
899auto AirIRGenerator::addConstant(Type type, uint64_t value) -> ExpressionType
900{
901 return addConstant(m_currentBlock, type, value);
902}
903
904auto AirIRGenerator::addConstant(BasicBlock* block, Type type, uint64_t value) -> ExpressionType
905{
906 auto result = tmpForType(type);
907 switch (type) {
908 case Type::I32:
909 case Type::I64:
910 case Type::Anyref:
911 append(block, Move, Arg::bigImm(value), result);
912 break;
913 case Type::F32:
914 case Type::F64: {
915 auto tmp = g64();
916 append(block, Move, Arg::bigImm(value), tmp);
917 append(block, type == Type::F32 ? Move32ToFloat : Move64ToDouble, tmp, result);
918 break;
919 }
920
921 default:
922 RELEASE_ASSERT_NOT_REACHED();
923 }
924
925 return result;
926}
927
928auto AirIRGenerator::addArguments(const Signature& signature) -> PartialResult
929{
930 RELEASE_ASSERT(m_locals.size() == signature.argumentCount()); // We handle arguments in the prologue
931 return { };
932}
933
934auto AirIRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
935{
936 ASSERT(value.tmp());
937 result = tmpForType(Type::I32);
938 auto tmp = g64();
939
940 append(Move, Arg::bigImm(JSValue::encode(jsNull())), tmp);
941 append(Compare64, Arg::relCond(MacroAssembler::Equal), value, tmp, result);
942
943 return { };
944}
945
946auto AirIRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
947{
948 ASSERT(m_locals[index].tmp());
949 result = tmpForType(m_locals[index].type());
950 append(moveOpForValueType(m_locals[index].type()), m_locals[index].tmp(), result);
951 return { };
952}
953
954auto AirIRGenerator::addUnreachable() -> PartialResult
955{
956 B3::PatchpointValue* unreachable = addPatchpoint(B3::Void);
957 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
958 this->emitThrowException(jit, ExceptionType::Unreachable);
959 });
960 unreachable->effects.terminal = true;
961 emitPatchpoint(unreachable, Tmp());
962 return { };
963}
964
965auto AirIRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
966{
967 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
968 instance->storeTopCallFrame(callFrame);
969
970 if (delta < 0)
971 return -1;
972
973 auto grown = instance->memory()->grow(PageCount(delta));
974 if (!grown) {
975 switch (grown.error()) {
976 case Memory::GrowFailReason::InvalidDelta:
977 case Memory::GrowFailReason::InvalidGrowSize:
978 case Memory::GrowFailReason::WouldExceedMaximum:
979 case Memory::GrowFailReason::OutOfMemory:
980 return -1;
981 }
982 RELEASE_ASSERT_NOT_REACHED();
983 }
984
985 return grown.value().pageCount();
986 };
987
988 result = g32();
989 emitCCall(growMemory, result, TypedTmp { Tmp(GPRInfo::callFrameRegister), Type::I64 }, instanceValue(), delta);
990 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_currentBlock);
991
992 return { };
993}
994
995auto AirIRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
996{
997 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
998
999 auto temp1 = g64();
1000 auto temp2 = g64();
1001
1002 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedMemorySize(), B3::Width64));
1003 append(Move, Arg::addr(instanceValue(), Instance::offsetOfCachedMemorySize()), temp1);
1004 constexpr uint32_t shiftValue = 16;
1005 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
1006 append(Move, Arg::imm(16), temp2);
1007 addShift(Type::I32, Urshift64, temp1, temp2, result);
1008 append(Move32, result, result);
1009
1010 return { };
1011}
1012
1013auto AirIRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
1014{
1015 ASSERT(m_locals[index].tmp());
1016 append(moveOpForValueType(m_locals[index].type()), value, m_locals[index].tmp());
1017 return { };
1018}
1019
1020auto AirIRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
1021{
1022 Type type = m_info.globals[index].type;
1023
1024 result = tmpForType(type);
1025
1026 auto temp = g64();
1027
1028 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1029 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1030
1031 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1032 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1033 append(moveOpForValueType(type), Arg::addr(temp, offset), result);
1034 else {
1035 auto temp2 = g64();
1036 append(Move, Arg::bigImm(offset), temp2);
1037 append(Add64, temp2, temp, temp);
1038 append(moveOpForValueType(type), Arg::addr(temp), result);
1039 }
1040 return { };
1041}
1042
1043auto AirIRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
1044{
1045 auto temp = g64();
1046
1047 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1048 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1049
1050 Type type = m_info.globals[index].type;
1051
1052 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1053 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1054 append(moveOpForValueType(type), value, Arg::addr(temp, offset));
1055 else {
1056 auto temp2 = g64();
1057 append(Move, Arg::bigImm(offset), temp2);
1058 append(Add64, temp2, temp, temp);
1059 append(moveOpForValueType(type), value, Arg::addr(temp));
1060 }
1061
1062 return { };
1063}
1064
1065inline AirIRGenerator::ExpressionType AirIRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
1066{
1067 ASSERT(m_memoryBaseGPR);
1068
1069 auto result = g64();
1070 append(Move32, pointer, result);
1071
1072 switch (m_mode) {
1073 case MemoryMode::BoundsChecking: {
1074 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
1075 ASSERT(m_memorySizeGPR);
1076 ASSERT(sizeOfOperation + offset > offset);
1077 auto temp = g64();
1078 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1079 append(Add64, result, temp);
1080
1081 emitCheck([&] {
1082 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, Tmp(m_memorySizeGPR));
1083 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1084 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1085 });
1086 break;
1087 }
1088
1089 case MemoryMode::Signaling: {
1090 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
1091 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
1092 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
1093 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
1094 // register + large offset access. We don't think this will be generated frequently.
1095 //
1096 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
1097 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
1098 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
1099 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
1100 if (offset >= Memory::fastMappedRedzoneBytes()) {
1101 uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
1102 auto temp = g64();
1103 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1104 append(Add64, result, temp);
1105 auto sizeMax = addConstant(Type::I64, maximum);
1106
1107 emitCheck([&] {
1108 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, sizeMax);
1109 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1110 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1111 });
1112 }
1113 break;
1114 }
1115 }
1116
1117 append(Add64, Tmp(m_memoryBaseGPR), result);
1118 return result;
1119}
1120
1121inline uint32_t sizeOfLoadOp(LoadOpType op)
1122{
1123 switch (op) {
1124 case LoadOpType::I32Load8S:
1125 case LoadOpType::I32Load8U:
1126 case LoadOpType::I64Load8S:
1127 case LoadOpType::I64Load8U:
1128 return 1;
1129 case LoadOpType::I32Load16S:
1130 case LoadOpType::I64Load16S:
1131 case LoadOpType::I32Load16U:
1132 case LoadOpType::I64Load16U:
1133 return 2;
1134 case LoadOpType::I32Load:
1135 case LoadOpType::I64Load32S:
1136 case LoadOpType::I64Load32U:
1137 case LoadOpType::F32Load:
1138 return 4;
1139 case LoadOpType::I64Load:
1140 case LoadOpType::F64Load:
1141 return 8;
1142 }
1143 RELEASE_ASSERT_NOT_REACHED();
1144}
1145
1146inline TypedTmp AirIRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
1147{
1148 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1149
1150 TypedTmp immTmp;
1151 TypedTmp newPtr;
1152 TypedTmp result;
1153
1154 Arg addrArg;
1155 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfLoadOp(op))))
1156 addrArg = Arg::addr(pointer, offset);
1157 else {
1158 immTmp = g64();
1159 newPtr = g64();
1160 append(Move, Arg::bigImm(offset), immTmp);
1161 append(Add64, immTmp, pointer, newPtr);
1162 addrArg = Arg::addr(newPtr);
1163 }
1164
1165 switch (op) {
1166 case LoadOpType::I32Load8S: {
1167 result = g32();
1168 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1169 break;
1170 }
1171
1172 case LoadOpType::I64Load8S: {
1173 result = g64();
1174 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1175 append(SignExtend32ToPtr, result, result);
1176 break;
1177 }
1178
1179 case LoadOpType::I32Load8U: {
1180 result = g32();
1181 appendEffectful(Load8, addrArg, result);
1182 break;
1183 }
1184
1185 case LoadOpType::I64Load8U: {
1186 result = g64();
1187 appendEffectful(Load8, addrArg, result);
1188 break;
1189 }
1190
1191 case LoadOpType::I32Load16S: {
1192 result = g32();
1193 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1194 break;
1195 }
1196
1197 case LoadOpType::I64Load16S: {
1198 result = g64();
1199 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1200 append(SignExtend32ToPtr, result, result);
1201 break;
1202 }
1203
1204 case LoadOpType::I32Load16U: {
1205 result = g32();
1206 appendEffectful(Load16, addrArg, result);
1207 break;
1208 }
1209
1210 case LoadOpType::I64Load16U: {
1211 result = g64();
1212 appendEffectful(Load16, addrArg, result);
1213 break;
1214 }
1215
1216 case LoadOpType::I32Load:
1217 result = g32();
1218 appendEffectful(Move32, addrArg, result);
1219 break;
1220
1221 case LoadOpType::I64Load32U: {
1222 result = g64();
1223 appendEffectful(Move32, addrArg, result);
1224 break;
1225 }
1226
1227 case LoadOpType::I64Load32S: {
1228 result = g64();
1229 appendEffectful(Move32, addrArg, result);
1230 append(SignExtend32ToPtr, result, result);
1231 break;
1232 }
1233
1234 case LoadOpType::I64Load: {
1235 result = g64();
1236 appendEffectful(Move, addrArg, result);
1237 break;
1238 }
1239
1240 case LoadOpType::F32Load: {
1241 result = f32();
1242 appendEffectful(MoveFloat, addrArg, result);
1243 break;
1244 }
1245
1246 case LoadOpType::F64Load: {
1247 result = f64();
1248 appendEffectful(MoveDouble, addrArg, result);
1249 break;
1250 }
1251 }
1252
1253 return result;
1254}
1255
1256auto AirIRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
1257{
1258 ASSERT(pointer.tmp().isGP());
1259
1260 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
1261 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1262 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1263 auto* patch = addPatchpoint(B3::Void);
1264 patch->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1265 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1266 });
1267 emitPatchpoint(patch, Tmp());
1268
1269 // We won't reach here, so we just pick a random reg.
1270 switch (op) {
1271 case LoadOpType::I32Load8S:
1272 case LoadOpType::I32Load16S:
1273 case LoadOpType::I32Load:
1274 case LoadOpType::I32Load16U:
1275 case LoadOpType::I32Load8U:
1276 result = g32();
1277 break;
1278 case LoadOpType::I64Load8S:
1279 case LoadOpType::I64Load8U:
1280 case LoadOpType::I64Load16S:
1281 case LoadOpType::I64Load32U:
1282 case LoadOpType::I64Load32S:
1283 case LoadOpType::I64Load:
1284 case LoadOpType::I64Load16U:
1285 result = g64();
1286 break;
1287 case LoadOpType::F32Load:
1288 result = f32();
1289 break;
1290 case LoadOpType::F64Load:
1291 result = f64();
1292 break;
1293 }
1294 } else
1295 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
1296
1297 return { };
1298}
1299
1300inline uint32_t sizeOfStoreOp(StoreOpType op)
1301{
1302 switch (op) {
1303 case StoreOpType::I32Store8:
1304 case StoreOpType::I64Store8:
1305 return 1;
1306 case StoreOpType::I32Store16:
1307 case StoreOpType::I64Store16:
1308 return 2;
1309 case StoreOpType::I32Store:
1310 case StoreOpType::I64Store32:
1311 case StoreOpType::F32Store:
1312 return 4;
1313 case StoreOpType::I64Store:
1314 case StoreOpType::F64Store:
1315 return 8;
1316 }
1317 RELEASE_ASSERT_NOT_REACHED();
1318}
1319
1320
1321inline void AirIRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1322{
1323 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1324
1325 TypedTmp immTmp;
1326 TypedTmp newPtr;
1327
1328 Arg addrArg;
1329 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfStoreOp(op))))
1330 addrArg = Arg::addr(pointer, offset);
1331 else {
1332 immTmp = g64();
1333 newPtr = g64();
1334 append(Move, Arg::bigImm(offset), immTmp);
1335 append(Add64, immTmp, pointer, newPtr);
1336 addrArg = Arg::addr(newPtr);
1337 }
1338
1339 switch (op) {
1340 case StoreOpType::I64Store8:
1341 case StoreOpType::I32Store8:
1342 append(Store8, value, addrArg);
1343 return;
1344
1345 case StoreOpType::I64Store16:
1346 case StoreOpType::I32Store16:
1347 append(Store16, value, addrArg);
1348 return;
1349
1350 case StoreOpType::I64Store32:
1351 case StoreOpType::I32Store:
1352 append(Move32, value, addrArg);
1353 return;
1354
1355 case StoreOpType::I64Store:
1356 append(Move, value, addrArg);
1357 return;
1358
1359 case StoreOpType::F32Store:
1360 append(MoveFloat, value, addrArg);
1361 return;
1362
1363 case StoreOpType::F64Store:
1364 append(MoveDouble, value, addrArg);
1365 return;
1366 }
1367
1368 RELEASE_ASSERT_NOT_REACHED();
1369}
1370
1371auto AirIRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1372{
1373 ASSERT(pointer.tmp().isGP());
1374
1375 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1376 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1377 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1378 auto* throwException = addPatchpoint(B3::Void);
1379 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1380 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1381 });
1382 emitPatchpoint(throwException, Tmp());
1383 } else
1384 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1385
1386 return { };
1387}
1388
1389auto AirIRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1390{
1391 ASSERT(nonZero.type() == zero.type());
1392 result = tmpForType(nonZero.type());
1393 append(moveOpForValueType(nonZero.type()), nonZero, result);
1394
1395 BasicBlock* isZero = m_code.addBlock();
1396 BasicBlock* continuation = m_code.addBlock();
1397
1398 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), condition, condition);
1399 m_currentBlock->setSuccessors(isZero, continuation);
1400
1401 append(isZero, moveOpForValueType(zero.type()), zero, result);
1402 append(isZero, Jump);
1403 isZero->setSuccessors(continuation);
1404
1405 m_currentBlock = continuation;
1406
1407 return { };
1408}
1409
1410void AirIRGenerator::emitTierUpCheck(uint32_t decrementCount, B3::Origin origin)
1411{
1412 UNUSED_PARAM(origin);
1413
1414 if (!m_tierUp)
1415 return;
1416
1417 auto countdownPtr = g64();
1418 auto oldCountdown = g64();
1419 auto newCountdown = g64();
1420
1421 append(Move, Arg::bigImm(reinterpret_cast<uint64_t>(m_tierUp)), countdownPtr);
1422 append(Move32, Arg::addr(countdownPtr), oldCountdown);
1423
1424 RELEASE_ASSERT(Arg::isValidImmForm(decrementCount));
1425 append(Move32, oldCountdown, newCountdown);
1426 append(Sub32, Arg::imm(decrementCount), newCountdown);
1427 append(Move32, newCountdown, Arg::addr(countdownPtr));
1428
1429 auto* patch = addPatchpoint(B3::Void);
1430 B3::Effects effects = B3::Effects::none();
1431 effects.reads = B3::HeapRange::top();
1432 effects.writes = B3::HeapRange::top();
1433 patch->effects = effects;
1434
1435 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1436 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
1437 MacroAssembler::Label tierUpResume = jit.label();
1438
1439 params.addLatePath([=] (CCallHelpers& jit) {
1440 tierUp.link(&jit);
1441
1442 const unsigned extraPaddingBytes = 0;
1443 RegisterSet registersToSpill = { };
1444 registersToSpill.add(GPRInfo::argumentGPR1);
1445 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1446
1447 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1448 MacroAssembler::Call call = jit.nearCall();
1449
1450 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1451 jit.jump(tierUpResume);
1452
1453 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1454 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
1455
1456 });
1457 });
1458 });
1459
1460 emitPatchpoint(patch, Tmp(), newCountdown, oldCountdown);
1461}
1462
1463AirIRGenerator::ControlData AirIRGenerator::addLoop(Type signature)
1464{
1465 BasicBlock* body = m_code.addBlock();
1466 BasicBlock* continuation = m_code.addBlock();
1467
1468 append(Jump);
1469 m_currentBlock->setSuccessors(body);
1470
1471 m_currentBlock = body;
1472 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
1473
1474 return ControlData(origin(), signature, tmpForType(signature), BlockType::Loop, continuation, body);
1475}
1476
1477AirIRGenerator::ControlData AirIRGenerator::addTopLevel(Type signature)
1478{
1479 return ControlData(B3::Origin(), signature, tmpForType(signature), BlockType::TopLevel, m_code.addBlock());
1480}
1481
1482AirIRGenerator::ControlData AirIRGenerator::addBlock(Type signature)
1483{
1484 return ControlData(origin(), signature, tmpForType(signature), BlockType::Block, m_code.addBlock());
1485}
1486
1487auto AirIRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
1488{
1489 BasicBlock* taken = m_code.addBlock();
1490 BasicBlock* notTaken = m_code.addBlock();
1491 BasicBlock* continuation = m_code.addBlock();
1492
1493 // Wasm bools are i32.
1494 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1495 m_currentBlock->setSuccessors(taken, notTaken);
1496
1497 m_currentBlock = taken;
1498 result = ControlData(origin(), signature, tmpForType(signature), BlockType::If, continuation, notTaken);
1499 return { };
1500}
1501
1502auto AirIRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1503{
1504 unifyValuesWithBlock(currentStack, data.result);
1505 append(Jump);
1506 m_currentBlock->setSuccessors(data.continuation);
1507 return addElseToUnreachable(data);
1508}
1509
1510auto AirIRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1511{
1512 ASSERT(data.type() == BlockType::If);
1513 m_currentBlock = data.special;
1514 data.convertIfToBlock();
1515 return { };
1516}
1517
1518auto AirIRGenerator::addReturn(const ControlData& data, const ExpressionList& returnValues) -> PartialResult
1519{
1520 ASSERT(returnValues.size() <= 1);
1521 if (returnValues.size()) {
1522 Tmp returnValueGPR = Tmp(GPRInfo::returnValueGPR);
1523 Tmp returnValueFPR = Tmp(FPRInfo::returnValueFPR);
1524 switch (data.signature()) {
1525 case Type::I32:
1526 append(Move32, returnValues[0], returnValueGPR);
1527 append(Ret32, returnValueGPR);
1528 break;
1529 case Type::I64:
1530 case Type::Anyref:
1531 append(Move, returnValues[0], returnValueGPR);
1532 append(Ret64, returnValueGPR);
1533 break;
1534 case Type::F32:
1535 append(MoveFloat, returnValues[0], returnValueFPR);
1536 append(RetFloat, returnValueFPR);
1537 break;
1538 case Type::F64:
1539 append(MoveDouble, returnValues[0], returnValueFPR);
1540 append(RetFloat, returnValueFPR);
1541 break;
1542 default:
1543 RELEASE_ASSERT_NOT_REACHED();
1544 }
1545 } else
1546 append(RetVoid);
1547 return { };
1548}
1549
1550// NOTE: All branches in Wasm are on 32-bit ints
1551
1552auto AirIRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1553{
1554 unifyValuesWithBlock(returnValues, data.resultForBranch());
1555
1556 BasicBlock* target = data.targetBlockForBranch();
1557 if (condition) {
1558 BasicBlock* continuation = m_code.addBlock();
1559 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1560 m_currentBlock->setSuccessors(target, continuation);
1561 m_currentBlock = continuation;
1562 } else {
1563 append(Jump);
1564 m_currentBlock->setSuccessors(target);
1565 }
1566
1567 return { };
1568}
1569
1570auto AirIRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1571{
1572 auto& successors = m_currentBlock->successors();
1573 ASSERT(successors.isEmpty());
1574 for (const auto& target : targets) {
1575 unifyValuesWithBlock(expressionStack, target->resultForBranch());
1576 successors.append(target->targetBlockForBranch());
1577 }
1578 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1579 successors.append(defaultTarget.targetBlockForBranch());
1580
1581 ASSERT(condition.type() == Type::I32);
1582
1583 // FIXME: We should consider dynamically switching between a jump table
1584 // and a binary switch depending on the number of successors.
1585 // https://bugs.webkit.org/show_bug.cgi?id=194477
1586
1587 size_t numTargets = targets.size();
1588
1589 auto* patchpoint = addPatchpoint(B3::Void);
1590 patchpoint->effects = B3::Effects::none();
1591 patchpoint->effects.terminal = true;
1592 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1593
1594 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1595 AllowMacroScratchRegisterUsage allowScratch(jit);
1596
1597 Vector<int64_t> cases;
1598 cases.reserveInitialCapacity(numTargets);
1599 for (size_t i = 0; i < numTargets; ++i)
1600 cases.uncheckedAppend(i);
1601
1602 GPRReg valueReg = params[0].gpr();
1603 BinarySwitch binarySwitch(valueReg, cases, BinarySwitch::Int32);
1604
1605 Vector<CCallHelpers::Jump> caseJumps;
1606 caseJumps.resize(numTargets);
1607
1608 while (binarySwitch.advance(jit)) {
1609 unsigned value = binarySwitch.caseValue();
1610 unsigned index = binarySwitch.caseIndex();
1611 ASSERT_UNUSED(value, value == index);
1612 ASSERT(index < numTargets);
1613 caseJumps[index] = jit.jump();
1614 }
1615
1616 CCallHelpers::JumpList fallThrough = binarySwitch.fallThrough();
1617
1618 Vector<Box<CCallHelpers::Label>> successorLabels = params.successorLabels();
1619 ASSERT(successorLabels.size() == caseJumps.size() + 1);
1620
1621 params.addLatePath([=, caseJumps = WTFMove(caseJumps), successorLabels = WTFMove(successorLabels)] (CCallHelpers& jit) {
1622 for (size_t i = 0; i < numTargets; ++i)
1623 caseJumps[i].linkTo(*successorLabels[i], &jit);
1624 fallThrough.linkTo(*successorLabels[numTargets], &jit);
1625 });
1626 });
1627
1628 emitPatchpoint(patchpoint, TypedTmp(), condition);
1629
1630 return { };
1631}
1632
1633auto AirIRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1634{
1635 ControlData& data = entry.controlData;
1636
1637 unifyValuesWithBlock(expressionStack, data.result);
1638 append(Jump);
1639 m_currentBlock->setSuccessors(data.continuation);
1640
1641 return addEndToUnreachable(entry);
1642}
1643
1644
1645auto AirIRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1646{
1647 ControlData& data = entry.controlData;
1648 m_currentBlock = data.continuation;
1649
1650 if (data.type() == BlockType::If) {
1651 append(data.special, Jump);
1652 data.special->setSuccessors(m_currentBlock);
1653 }
1654
1655 for (const auto& result : data.result)
1656 entry.enclosedExpressionStack.append(result);
1657
1658 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1659 if (data.type() == BlockType::TopLevel)
1660 return addReturn(data, entry.enclosedExpressionStack);
1661
1662 return { };
1663}
1664
1665auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1666{
1667 ASSERT(signature.argumentCount() == args.size());
1668
1669 m_makesCalls = true;
1670
1671 Type returnType = signature.returnType();
1672 if (returnType != Type::Void)
1673 result = tmpForType(returnType);
1674
1675 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1676
1677 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1678 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1679
1680 auto currentInstance = g64();
1681 append(Move, instanceValue(), currentInstance);
1682
1683 auto targetInstance = g64();
1684
1685 // FIXME: We should have better isel here.
1686 // https://bugs.webkit.org/show_bug.cgi?id=193999
1687 append(Move, Arg::bigImm(Instance::offsetOfTargetInstance(functionIndex)), targetInstance);
1688 append(Add64, instanceValue(), targetInstance);
1689 append(Move, Arg::addr(targetInstance), targetInstance);
1690
1691 BasicBlock* isWasmBlock = m_code.addBlock();
1692 BasicBlock* isEmbedderBlock = m_code.addBlock();
1693 BasicBlock* continuation = m_code.addBlock();
1694
1695 append(BranchTest64, Arg::resCond(MacroAssembler::NonZero), targetInstance, targetInstance);
1696 m_currentBlock->setSuccessors(isWasmBlock, isEmbedderBlock);
1697
1698 {
1699 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1700 patchpoint->effects.writesPinned = true;
1701 patchpoint->effects.readsPinned = true;
1702 // We need to clobber all potential pinned registers since we might be leaving the instance.
1703 // We pessimistically assume we could be calling to something that is bounds checking.
1704 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1705 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1706
1707 Vector<ConstrainedTmp> patchArgs;
1708 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1709 patchArgs.append({ tmp, rep });
1710 });
1711
1712 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1713 AllowMacroScratchRegisterUsage allowScratch(jit);
1714 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1715 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1716 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1717 });
1718 });
1719
1720 emitPatchpoint(isWasmBlock, patchpoint, result, WTFMove(patchArgs));
1721 append(isWasmBlock, Jump);
1722 isWasmBlock->setSuccessors(continuation);
1723 }
1724
1725 {
1726 auto jumpDestination = g64();
1727 append(isEmbedderBlock, Move, Arg::bigImm(Instance::offsetOfWasmToEmbedderStub(functionIndex)), jumpDestination);
1728 append(isEmbedderBlock, Add64, instanceValue(), jumpDestination);
1729 append(isEmbedderBlock, Move, Arg::addr(jumpDestination), jumpDestination);
1730
1731 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1732 patchpoint->effects.writesPinned = true;
1733 patchpoint->effects.readsPinned = true;
1734 // We need to clobber all potential pinned registers since we might be leaving the instance.
1735 // We pessimistically assume we could be calling to something that is bounds checking.
1736 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1737 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1738
1739 Vector<ConstrainedTmp> patchArgs;
1740 patchArgs.append(jumpDestination);
1741
1742 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1743 patchArgs.append({ tmp, rep });
1744 });
1745
1746 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1747 AllowMacroScratchRegisterUsage allowScratch(jit);
1748 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1749 });
1750
1751 emitPatchpoint(isEmbedderBlock, patchpoint, result, WTFMove(patchArgs));
1752 append(isEmbedderBlock, Jump);
1753 isEmbedderBlock->setSuccessors(continuation);
1754 }
1755
1756 m_currentBlock = continuation;
1757 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1758 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, continuation);
1759 } else {
1760 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1761 patchpoint->effects.writesPinned = true;
1762 patchpoint->effects.readsPinned = true;
1763
1764 Vector<ConstrainedTmp> patchArgs;
1765 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1766 patchArgs.append({ tmp, rep });
1767 });
1768
1769 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1770 AllowMacroScratchRegisterUsage allowScratch(jit);
1771 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1772 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1773 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1774 });
1775 });
1776
1777 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(patchArgs));
1778 }
1779
1780 return { };
1781}
1782
1783auto AirIRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1784{
1785 ExpressionType calleeIndex = args.takeLast();
1786 ASSERT(signature.argumentCount() == args.size());
1787
1788 m_makesCalls = true;
1789 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1790 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1791 // can be to the embedder for our stack check calculation.
1792 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1793
1794 auto currentInstance = g64();
1795 append(Move, instanceValue(), currentInstance);
1796
1797 ExpressionType callableFunctionBuffer = g64();
1798 ExpressionType instancesBuffer = g64();
1799 ExpressionType callableFunctionBufferLength = g64();
1800 {
1801 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfTable(), B3::Width64));
1802 RELEASE_ASSERT(Arg::isValidAddrForm(Table::offsetOfFunctions(), B3::Width64));
1803 RELEASE_ASSERT(Arg::isValidAddrForm(Table::offsetOfInstances(), B3::Width64));
1804 RELEASE_ASSERT(Arg::isValidAddrForm(Table::offsetOfLength(), B3::Width64));
1805
1806 append(Move, Arg::addr(instanceValue(), Instance::offsetOfTable()), callableFunctionBufferLength);
1807 append(Move, Arg::addr(callableFunctionBufferLength, Table::offsetOfFunctions()), callableFunctionBuffer);
1808 append(Move, Arg::addr(callableFunctionBufferLength, Table::offsetOfInstances()), instancesBuffer);
1809 append(Move32, Arg::addr(callableFunctionBufferLength, Table::offsetOfLength()), callableFunctionBufferLength);
1810 }
1811
1812 append(Move32, calleeIndex, calleeIndex);
1813
1814 // Check the index we are looking for is valid.
1815 emitCheck([&] {
1816 return Inst(Branch32, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), calleeIndex, callableFunctionBufferLength);
1817 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1818 this->emitThrowException(jit, ExceptionType::OutOfBoundsCallIndirect);
1819 });
1820
1821 ExpressionType calleeCode = g64();
1822 {
1823 ExpressionType calleeSignatureIndex = g64();
1824 // Compute the offset in the table index space we are looking for.
1825 append(Move, Arg::imm(sizeof(WasmToWasmImportableFunction)), calleeSignatureIndex);
1826 append(Mul64, calleeIndex, calleeSignatureIndex);
1827 append(Add64, callableFunctionBuffer, calleeSignatureIndex);
1828
1829 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation()), calleeCode); // Pointer to callee code.
1830
1831 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1832 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1833 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1834
1835 // FIXME: This seems dumb to do two checks just for a nicer error message.
1836 // We should move just to use a single branch and then figure out what
1837 // error to use in the exception handler.
1838
1839 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfSignatureIndex()), calleeSignatureIndex);
1840
1841 emitCheck([&] {
1842 static_assert(Signature::invalidIndex == 0, "");
1843 return Inst(BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), calleeSignatureIndex, calleeSignatureIndex);
1844 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1845 this->emitThrowException(jit, ExceptionType::NullTableEntry);
1846 });
1847
1848 ExpressionType expectedSignatureIndex = g64();
1849 append(Move, Arg::bigImm(SignatureInformation::get(signature)), expectedSignatureIndex);
1850 emitCheck([&] {
1851 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::NotEqual), calleeSignatureIndex, expectedSignatureIndex);
1852 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1853 this->emitThrowException(jit, ExceptionType::BadSignature);
1854 });
1855 }
1856
1857 // Do a context switch if needed.
1858 {
1859 auto newContextInstance = g64();
1860 append(Move, Arg::index(instancesBuffer, calleeIndex, 8, 0), newContextInstance);
1861
1862 BasicBlock* doContextSwitch = m_code.addBlock();
1863 BasicBlock* continuation = m_code.addBlock();
1864
1865 append(Branch64, Arg::relCond(MacroAssembler::Equal), newContextInstance, instanceValue());
1866 m_currentBlock->setSuccessors(continuation, doContextSwitch);
1867
1868 auto* patchpoint = addPatchpoint(B3::Void);
1869 patchpoint->effects.writesPinned = true;
1870 // We pessimistically assume we're calling something with BoundsChecking memory.
1871 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1872 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1873 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1874 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
1875
1876 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1877 AllowMacroScratchRegisterUsage allowScratch(jit);
1878 GPRReg newContextInstance = params[0].gpr();
1879 GPRReg oldContextInstance = params[1].gpr();
1880 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1881 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1882 ASSERT(newContextInstance != baseMemory);
1883 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1884 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1885 jit.storeWasmContextInstance(newContextInstance);
1886 // FIXME: We should support more than one memory size register
1887 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
1888 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1889 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
1890
1891 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1892 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1893
1894 jit.cageConditionally(Gigacage::Primitive, baseMemory, scratchOrSize);
1895 });
1896
1897 emitPatchpoint(doContextSwitch, patchpoint, Tmp(), newContextInstance, instanceValue());
1898 append(doContextSwitch, Jump);
1899 doContextSwitch->setSuccessors(continuation);
1900
1901 m_currentBlock = continuation;
1902 }
1903
1904 append(Move, Arg::addr(calleeCode), calleeCode);
1905
1906 Type returnType = signature.returnType();
1907 if (returnType != Type::Void)
1908 result = tmpForType(returnType);
1909
1910 auto* patch = addPatchpoint(toB3Type(returnType));
1911 patch->effects.writesPinned = true;
1912 patch->effects.readsPinned = true;
1913 // We need to clobber all potential pinned registers since we might be leaving the instance.
1914 // We pessimistically assume we're always calling something that is bounds checking so
1915 // because the wasm->wasm thunk unconditionally overrides the size registers.
1916 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1917 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1918 patch->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1919
1920 Vector<ConstrainedTmp> emitArgs;
1921 emitArgs.append(calleeCode);
1922 wasmCallingConventionAir().setupCall(m_code, returnType, patch, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1923 emitArgs.append({ tmp, rep });
1924 });
1925 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1926 AllowMacroScratchRegisterUsage allowScratch(jit);
1927 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1928 });
1929
1930 emitPatchpoint(m_currentBlock, patch, result, WTFMove(emitArgs));
1931
1932 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1933 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, m_currentBlock);
1934
1935 return { };
1936}
1937
1938void AirIRGenerator::unify(const ExpressionType& dst, const ExpressionType& source)
1939{
1940 ASSERT(dst.type() == source.type());
1941 append(moveOpForValueType(dst.type()), source, dst);
1942}
1943
1944void AirIRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1945{
1946 ASSERT(result.size() <= resultStack.size());
1947
1948 for (size_t i = 0; i < result.size(); ++i)
1949 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1950}
1951
1952void AirIRGenerator::dump(const Vector<ControlEntry>&, const ExpressionList*)
1953{
1954}
1955
1956auto AirIRGenerator::origin() -> B3::Origin
1957{
1958 // FIXME: We should implement a way to give Inst's an origin.
1959 return B3::Origin();
1960}
1961
1962Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileAir(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1963{
1964 auto result = std::make_unique<InternalFunction>();
1965
1966 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1967 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1968
1969 B3::Procedure procedure;
1970 Code& code = procedure.code();
1971
1972 procedure.setOriginPrinter([] (PrintStream& out, B3::Origin origin) {
1973 if (origin.data())
1974 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1975 });
1976
1977 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1978 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1979 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1980 // optLevel=1.
1981 procedure.setNeedsUsedRegisters(false);
1982
1983 procedure.setOptLevel(Options::webAssemblyBBQOptimizationLevel());
1984
1985 AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, throwWasmException, signature);
1986 FunctionParser<AirIRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1987 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1988
1989
1990 for (BasicBlock* block : code) {
1991 for (size_t i = 0; i < block->numSuccessors(); ++i)
1992 block->successorBlock(i)->addPredecessor(block);
1993 }
1994
1995 {
1996 B3::Air::prepareForGeneration(code);
1997 B3::Air::generate(code, *compilationContext.wasmEntrypointJIT);
1998 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1999 result->entrypoint.calleeSaveRegisters = code.calleeSaveRegisterAtOffsetList();
2000 }
2001
2002 return result;
2003}
2004
2005template <typename IntType>
2006void AirIRGenerator::emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right)
2007{
2008 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2009
2010 emitCheck([&] {
2011 return Inst(sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), right, right);
2012 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2013 this->emitThrowException(jit, ExceptionType::DivisionByZero);
2014 });
2015
2016 if (isSignedDiv) {
2017 ASSERT(std::is_signed<IntType>::value);
2018 IntType min = std::numeric_limits<IntType>::min();
2019
2020 // FIXME: Better isel for compare with imms here.
2021 // https://bugs.webkit.org/show_bug.cgi?id=193999
2022 auto minTmp = sizeof(IntType) == 4 ? g32() : g64();
2023 auto negOne = sizeof(IntType) == 4 ? g32() : g64();
2024
2025 B3::Air::Opcode op = sizeof(IntType) == 4 ? Compare32 : Compare64;
2026 append(Move, Arg::bigImm(static_cast<uint64_t>(min)), minTmp);
2027 append(op, Arg::relCond(MacroAssembler::Equal), left, minTmp, minTmp);
2028
2029 append(Move, Arg::imm(-1), negOne);
2030 append(op, Arg::relCond(MacroAssembler::Equal), right, negOne, negOne);
2031
2032 emitCheck([&] {
2033 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), minTmp, negOne);
2034 },
2035 [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2036 this->emitThrowException(jit, ExceptionType::IntegerOverflow);
2037 });
2038 }
2039}
2040
2041template <typename IntType>
2042void AirIRGenerator::emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result)
2043{
2044 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2045
2046 result = sizeof(IntType) == 4 ? g32() : g64();
2047
2048 bool isSigned = std::is_signed<IntType>::value;
2049
2050 if (isARM64()) {
2051 B3::Air::Opcode div;
2052 switch (sizeof(IntType)) {
2053 case 4:
2054 div = isSigned ? Div32 : UDiv32;
2055 break;
2056 case 8:
2057 div = isSigned ? Div64 : UDiv64;
2058 break;
2059 }
2060
2061 append(div, lhs, rhs, result);
2062
2063 if (!isDiv) {
2064 append(sizeof(IntType) == 4 ? Mul32 : Mul64, result, rhs, result);
2065 append(sizeof(IntType) == 4 ? Sub32 : Sub64, lhs, result, result);
2066 }
2067
2068 return;
2069 }
2070
2071#if CPU(X86) || CPU(X86_64)
2072 Tmp eax(X86Registers::eax);
2073 Tmp edx(X86Registers::edx);
2074
2075 if (isSigned) {
2076 B3::Air::Opcode convertToDoubleWord;
2077 B3::Air::Opcode div;
2078 switch (sizeof(IntType)) {
2079 case 4:
2080 convertToDoubleWord = X86ConvertToDoubleWord32;
2081 div = X86Div32;
2082 break;
2083 case 8:
2084 convertToDoubleWord = X86ConvertToQuadWord64;
2085 div = X86Div64;
2086 break;
2087 default:
2088 RELEASE_ASSERT_NOT_REACHED();
2089 }
2090
2091 // We implement "res = Div<Chill>/Mod<Chill>(num, den)" as follows:
2092 //
2093 // if (den + 1 <=_unsigned 1) {
2094 // if (!den) {
2095 // res = 0;
2096 // goto done;
2097 // }
2098 // if (num == -2147483648) {
2099 // res = isDiv ? num : 0;
2100 // goto done;
2101 // }
2102 // }
2103 // res = num (/ or %) dev;
2104 // done:
2105
2106 BasicBlock* denIsGood = m_code.addBlock();
2107 BasicBlock* denMayBeBad = m_code.addBlock();
2108 BasicBlock* denNotZero = m_code.addBlock();
2109 BasicBlock* continuation = m_code.addBlock();
2110
2111 auto temp = sizeof(IntType) == 4 ? g32() : g64();
2112 auto one = addConstant(sizeof(IntType) == 4 ? Type::I32 : Type::I64, 1);
2113
2114 append(sizeof(IntType) == 4 ? Add32 : Add64, rhs, one, temp);
2115 append(sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Above), temp, one);
2116 m_currentBlock->setSuccessors(denIsGood, denMayBeBad);
2117
2118 append(denMayBeBad, Xor64, result, result);
2119 append(denMayBeBad, sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, Arg::resCond(MacroAssembler::Zero), rhs, rhs);
2120 denMayBeBad->setSuccessors(continuation, denNotZero);
2121
2122 auto min = addConstant(denNotZero, sizeof(IntType) == 4 ? Type::I32 : Type::I64, std::numeric_limits<IntType>::min());
2123 if (isDiv)
2124 append(denNotZero, sizeof(IntType) == 4 ? Move32 : Move, min, result);
2125 else {
2126 // Result is zero, as set above...
2127 }
2128 append(denNotZero, sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Equal), lhs, min);
2129 denNotZero->setSuccessors(continuation, denIsGood);
2130
2131 auto divResult = isDiv ? eax : edx;
2132 append(denIsGood, Move, lhs, eax);
2133 append(denIsGood, convertToDoubleWord, eax, edx);
2134 append(denIsGood, div, eax, edx, rhs);
2135 append(denIsGood, sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2136 append(denIsGood, Jump);
2137 denIsGood->setSuccessors(continuation);
2138
2139 m_currentBlock = continuation;
2140 return;
2141 }
2142
2143 B3::Air::Opcode div = sizeof(IntType) == 4 ? X86UDiv32 : X86UDiv64;
2144
2145 Tmp divResult = isDiv ? eax : edx;
2146
2147 append(Move, lhs, eax);
2148 append(Xor64, edx, edx);
2149 append(div, eax, edx, rhs);
2150 append(sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2151#else
2152 RELEASE_ASSERT_NOT_REACHED();
2153#endif
2154}
2155
2156template<>
2157auto AirIRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2158{
2159 emitChecksForModOrDiv<int32_t>(true, left, right);
2160 emitModOrDiv<int32_t>(true, left, right, result);
2161 return { };
2162}
2163
2164template<>
2165auto AirIRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2166{
2167 emitChecksForModOrDiv<int32_t>(false, left, right);
2168 emitModOrDiv<int32_t>(false, left, right, result);
2169 return { };
2170}
2171
2172template<>
2173auto AirIRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2174{
2175 emitChecksForModOrDiv<uint32_t>(false, left, right);
2176 emitModOrDiv<uint32_t>(true, left, right, result);
2177 return { };
2178}
2179
2180template<>
2181auto AirIRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2182{
2183 emitChecksForModOrDiv<uint32_t>(false, left, right);
2184 emitModOrDiv<uint32_t>(false, left, right, result);
2185 return { };
2186}
2187
2188template<>
2189auto AirIRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2190{
2191 emitChecksForModOrDiv<int64_t>(true, left, right);
2192 emitModOrDiv<int64_t>(true, left, right, result);
2193 return { };
2194}
2195
2196template<>
2197auto AirIRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2198{
2199 emitChecksForModOrDiv<int64_t>(false, left, right);
2200 emitModOrDiv<int64_t>(false, left, right, result);
2201 return { };
2202}
2203
2204template<>
2205auto AirIRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2206{
2207 emitChecksForModOrDiv<uint64_t>(false, left, right);
2208 emitModOrDiv<uint64_t>(true, left, right, result);
2209 return { };
2210}
2211
2212template<>
2213auto AirIRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2214{
2215 emitChecksForModOrDiv<uint64_t>(false, left, right);
2216 emitModOrDiv<uint64_t>(false, left, right, result);
2217 return { };
2218}
2219
2220template<>
2221auto AirIRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2222{
2223 auto* patchpoint = addPatchpoint(B3::Int32);
2224 patchpoint->effects = B3::Effects::none();
2225 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2226 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
2227 });
2228 result = g32();
2229 emitPatchpoint(patchpoint, result, arg);
2230 return { };
2231}
2232
2233template<>
2234auto AirIRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2235{
2236 auto* patchpoint = addPatchpoint(B3::Int64);
2237 patchpoint->effects = B3::Effects::none();
2238 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2239 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
2240 });
2241 result = g64();
2242 emitPatchpoint(patchpoint, result, arg);
2243 return { };
2244}
2245
2246template<>
2247auto AirIRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2248{
2249 result = g32();
2250
2251#if CPU(X86_64)
2252 if (MacroAssembler::supportsCountPopulation()) {
2253 auto* patchpoint = addPatchpoint(B3::Int32);
2254 patchpoint->effects = B3::Effects::none();
2255 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2256 jit.countPopulation32(params[1].gpr(), params[0].gpr());
2257 });
2258 emitPatchpoint(patchpoint, result, arg);
2259 return { };
2260 }
2261#endif
2262
2263 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
2264 emitCCall(popcount, result, arg);
2265 return { };
2266}
2267
2268template<>
2269auto AirIRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2270{
2271 result = g64();
2272
2273#if CPU(X86_64)
2274 if (MacroAssembler::supportsCountPopulation()) {
2275 auto* patchpoint = addPatchpoint(B3::Int64);
2276 patchpoint->effects = B3::Effects::none();
2277 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2278 jit.countPopulation64(params[1].gpr(), params[0].gpr());
2279 });
2280 emitPatchpoint(patchpoint, result, arg);
2281 return { };
2282 }
2283#endif
2284
2285 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
2286 emitCCall(popcount, result, arg);
2287 return { };
2288}
2289
2290template<>
2291auto AirIRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2292{
2293 auto* patchpoint = addPatchpoint(B3::Double);
2294 patchpoint->effects = B3::Effects::none();
2295 if (isX86())
2296 patchpoint->numGPScratchRegisters = 1;
2297 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2298 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2299 AllowMacroScratchRegisterUsage allowScratch(jit);
2300#if CPU(X86_64)
2301 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2302#else
2303 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
2304#endif
2305 });
2306 result = f64();
2307 emitPatchpoint(patchpoint, result, arg);
2308 return { };
2309}
2310
2311template<>
2312auto AirIRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2313{
2314 auto* patchpoint = addPatchpoint(B3::Float);
2315 patchpoint->effects = B3::Effects::none();
2316 if (isX86())
2317 patchpoint->numGPScratchRegisters = 1;
2318 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2319 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2320 AllowMacroScratchRegisterUsage allowScratch(jit);
2321#if CPU(X86_64)
2322 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2323#else
2324 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
2325#endif
2326 });
2327 result = f32();
2328 emitPatchpoint(patchpoint, result, arg);
2329 return { };
2330}
2331
2332template<>
2333auto AirIRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2334{
2335 auto* patchpoint = addPatchpoint(B3::Double);
2336 patchpoint->effects = B3::Effects::none();
2337 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2338 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
2339 });
2340 result = f64();
2341 emitPatchpoint(patchpoint, result, arg);
2342 return { };
2343}
2344
2345template<>
2346auto AirIRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2347{
2348 auto* patchpoint = addPatchpoint(B3::Float);
2349 patchpoint->effects = B3::Effects::none();
2350 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2351 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
2352 });
2353 result = f32();
2354 emitPatchpoint(patchpoint, result, arg);
2355 return { };
2356}
2357
2358template<>
2359auto AirIRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2360{
2361 auto* patchpoint = addPatchpoint(B3::Double);
2362 patchpoint->effects = B3::Effects::none();
2363 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2364 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
2365 });
2366 result = f64();
2367 emitPatchpoint(patchpoint, result, arg);
2368 return { };
2369}
2370
2371template<>
2372auto AirIRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2373{
2374 auto* patchpoint = addPatchpoint(B3::Float);
2375 patchpoint->effects = B3::Effects::none();
2376 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2377 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
2378 });
2379 result = f32();
2380 emitPatchpoint(patchpoint, result, arg);
2381 return { };
2382}
2383
2384template<>
2385auto AirIRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2386{
2387 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
2388 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
2389
2390 auto temp1 = g32();
2391 auto temp2 = g32();
2392 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2393 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2394 append(Or32, temp1, temp2);
2395
2396 emitCheck([&] {
2397 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2398 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2399 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2400 });
2401
2402 auto* patchpoint = addPatchpoint(B3::Int32);
2403 patchpoint->effects = B3::Effects::none();
2404 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2405 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
2406 });
2407 result = g32();
2408 emitPatchpoint(patchpoint, result, arg);
2409
2410 return { };
2411}
2412
2413template<>
2414auto AirIRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2415{
2416 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
2417 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
2418
2419 auto temp1 = g32();
2420 auto temp2 = g32();
2421 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2422 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2423 append(Or32, temp1, temp2);
2424
2425 emitCheck([&] {
2426 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2427 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2428 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2429 });
2430
2431 auto* patchpoint = addPatchpoint(B3::Int32);
2432 patchpoint->effects = B3::Effects::none();
2433 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2434 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
2435 });
2436 result = g32();
2437 emitPatchpoint(patchpoint, result, arg);
2438 return { };
2439}
2440
2441
2442template<>
2443auto AirIRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2444{
2445 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
2446 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2447
2448 auto temp1 = g32();
2449 auto temp2 = g32();
2450 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2451 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2452 append(Or32, temp1, temp2);
2453
2454 emitCheck([&] {
2455 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2456 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2457 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2458 });
2459
2460 auto* patchpoint = addPatchpoint(B3::Int32);
2461 patchpoint->effects = B3::Effects::none();
2462 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2463 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
2464 });
2465 result = g32();
2466 emitPatchpoint(patchpoint, result, arg);
2467 return { };
2468}
2469
2470template<>
2471auto AirIRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2472{
2473 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
2474 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2475
2476 auto temp1 = g32();
2477 auto temp2 = g32();
2478 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2479 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2480 append(Or32, temp1, temp2);
2481
2482 emitCheck([&] {
2483 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2484 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2485 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2486 });
2487
2488 auto* patchpoint = addPatchpoint(B3::Int32);
2489 patchpoint->effects = B3::Effects::none();
2490 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2491 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
2492 });
2493 result = g32();
2494 emitPatchpoint(patchpoint, result, arg);
2495 return { };
2496}
2497
2498template<>
2499auto AirIRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2500{
2501 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
2502 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
2503
2504 auto temp1 = g32();
2505 auto temp2 = g32();
2506 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2507 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2508 append(Or32, temp1, temp2);
2509
2510 emitCheck([&] {
2511 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2512 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2513 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2514 });
2515
2516 auto* patchpoint = addPatchpoint(B3::Int64);
2517 patchpoint->effects = B3::Effects::none();
2518 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2519 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
2520 });
2521
2522 result = g64();
2523 emitPatchpoint(patchpoint, result, arg);
2524 return { };
2525}
2526
2527template<>
2528auto AirIRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2529{
2530 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
2531 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2532
2533 auto temp1 = g32();
2534 auto temp2 = g32();
2535 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2536 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2537 append(Or32, temp1, temp2);
2538
2539 emitCheck([&] {
2540 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2541 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2542 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2543 });
2544
2545 TypedTmp signBitConstant;
2546 if (isX86())
2547 signBitConstant = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2548
2549 Vector<ConstrainedTmp> args;
2550 auto* patchpoint = addPatchpoint(B3::Int64);
2551 patchpoint->effects = B3::Effects::none();
2552 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2553 args.append(arg);
2554 if (isX86()) {
2555 args.append(signBitConstant);
2556 patchpoint->numFPScratchRegisters = 1;
2557 }
2558 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2559 AllowMacroScratchRegisterUsage allowScratch(jit);
2560 FPRReg scratch = InvalidFPRReg;
2561 FPRReg constant = InvalidFPRReg;
2562 if (isX86()) {
2563 scratch = params.fpScratch(0);
2564 constant = params[2].fpr();
2565 }
2566 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2567 });
2568
2569 result = g64();
2570 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2571 return { };
2572}
2573
2574template<>
2575auto AirIRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2576{
2577 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2578 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2579
2580 auto temp1 = g32();
2581 auto temp2 = g32();
2582 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2583 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2584 append(Or32, temp1, temp2);
2585
2586 emitCheck([&] {
2587 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2588 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2589 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2590 });
2591
2592 auto* patchpoint = addPatchpoint(B3::Int64);
2593 patchpoint->effects = B3::Effects::none();
2594 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2595 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2596 });
2597 result = g64();
2598 emitPatchpoint(patchpoint, result, arg);
2599 return { };
2600}
2601
2602template<>
2603auto AirIRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2604{
2605 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2606 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2607
2608 auto temp1 = g32();
2609 auto temp2 = g32();
2610 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2611 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2612 append(Or32, temp1, temp2);
2613
2614 emitCheck([&] {
2615 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2616 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2617 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2618 });
2619
2620 TypedTmp signBitConstant;
2621 if (isX86())
2622 signBitConstant = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2623
2624 auto* patchpoint = addPatchpoint(B3::Int64);
2625 patchpoint->effects = B3::Effects::none();
2626 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2627 Vector<ConstrainedTmp> args;
2628 args.append(arg);
2629 if (isX86()) {
2630 args.append(signBitConstant);
2631 patchpoint->numFPScratchRegisters = 1;
2632 }
2633 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2634 AllowMacroScratchRegisterUsage allowScratch(jit);
2635 FPRReg scratch = InvalidFPRReg;
2636 FPRReg constant = InvalidFPRReg;
2637 if (isX86()) {
2638 scratch = params.fpScratch(0);
2639 constant = params[2].fpr();
2640 }
2641 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2642 });
2643
2644 result = g64();
2645 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2646
2647 return { };
2648}
2649
2650auto AirIRGenerator::addShift(Type type, B3::Air::Opcode op, ExpressionType value, ExpressionType shift, ExpressionType& result) -> PartialResult
2651{
2652 ASSERT(type == Type::I64 || type == Type::I32);
2653 result = tmpForType(type);
2654
2655 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2656 append(op, value, shift, result);
2657 return { };
2658 }
2659
2660#if CPU(X86_64)
2661 Tmp ecx = Tmp(X86Registers::ecx);
2662 append(Move, value, result);
2663 append(Move, shift, ecx);
2664 append(op, ecx, result);
2665#else
2666 RELEASE_ASSERT_NOT_REACHED();
2667#endif
2668 return { };
2669}
2670
2671auto AirIRGenerator::addIntegerSub(B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2672{
2673 ASSERT(op == Sub32 || op == Sub64);
2674
2675 result = op == Sub32 ? g32() : g64();
2676
2677 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2678 append(op, lhs, rhs, result);
2679 return { };
2680 }
2681
2682 RELEASE_ASSERT(isX86());
2683 // Sub a, b
2684 // means
2685 // b = b Sub a
2686 append(Move, lhs, result);
2687 append(op, rhs, result);
2688 return { };
2689}
2690
2691auto AirIRGenerator::addFloatingPointAbs(B3::Air::Opcode op, ExpressionType value, ExpressionType& result) -> PartialResult
2692{
2693 RELEASE_ASSERT(op == AbsFloat || op == AbsDouble);
2694
2695 result = op == AbsFloat ? f32() : f64();
2696
2697 if (isValidForm(op, Arg::Tmp, Arg::Tmp)) {
2698 append(op, value, result);
2699 return { };
2700 }
2701
2702 RELEASE_ASSERT(isX86());
2703
2704 if (op == AbsFloat) {
2705 auto constant = g32();
2706 append(Move, Arg::imm(static_cast<uint32_t>(~(1ull << 31))), constant);
2707 append(Move32ToFloat, constant, result);
2708 append(AndFloat, value, result);
2709 } else {
2710 auto constant = g64();
2711 append(Move, Arg::bigImm(~(1ull << 63)), constant);
2712 append(Move64ToDouble, constant, result);
2713 append(AndDouble, value, result);
2714 }
2715 return { };
2716}
2717
2718auto AirIRGenerator::addFloatingPointBinOp(Type type, B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2719{
2720 ASSERT(type == Type::F32 || type == Type::F64);
2721 result = tmpForType(type);
2722
2723 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2724 append(op, lhs, rhs, result);
2725 return { };
2726 }
2727
2728 RELEASE_ASSERT(isX86());
2729
2730 // Op a, b
2731 // means
2732 // b = b Op a
2733 append(moveOpForValueType(type), lhs, result);
2734 append(op, rhs, result);
2735 return { };
2736}
2737
2738template<> auto AirIRGenerator::addOp<OpType::F32Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2739{
2740 result = f32();
2741 append(CeilFloat, arg0, result);
2742 return { };
2743}
2744
2745template<> auto AirIRGenerator::addOp<OpType::I32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2746{
2747 result = g32();
2748 append(Mul32, arg0, arg1, result);
2749 return { };
2750}
2751
2752template<> auto AirIRGenerator::addOp<OpType::I32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2753{
2754 return addIntegerSub(Sub32, arg0, arg1, result);
2755}
2756
2757template<> auto AirIRGenerator::addOp<OpType::F64Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2758{
2759 result = g32();
2760 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
2761 return { };
2762}
2763
2764template<> auto AirIRGenerator::addOp<OpType::F32DemoteF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2765{
2766 result = f32();
2767 append(ConvertDoubleToFloat, arg0, result);
2768 return { };
2769}
2770
2771template<> auto AirIRGenerator::addOp<OpType::F32Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2772{
2773 return addFloatingPointMinOrMax(F32, MinOrMax::Min, arg0, arg1, result);
2774}
2775
2776template<> auto AirIRGenerator::addOp<OpType::F64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2777{
2778 result = g32();
2779 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
2780 return { };
2781}
2782
2783template<> auto AirIRGenerator::addOp<OpType::F64Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2784{
2785 result = g32();
2786 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
2787 return { };
2788}
2789
2790auto AirIRGenerator::addFloatingPointMinOrMax(Type floatType, MinOrMax minOrMax, ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2791{
2792 ASSERT(floatType == F32 || floatType == F64);
2793 result = tmpForType(floatType);
2794
2795 BasicBlock* isEqual = m_code.addBlock();
2796 BasicBlock* notEqual = m_code.addBlock();
2797 BasicBlock* isLessThan = m_code.addBlock();
2798 BasicBlock* notLessThan = m_code.addBlock();
2799 BasicBlock* isGreaterThan = m_code.addBlock();
2800 BasicBlock* isNaN = m_code.addBlock();
2801 BasicBlock* continuation = m_code.addBlock();
2802
2803 auto branchOp = floatType == F32 ? BranchFloat : BranchDouble;
2804 append(m_currentBlock, branchOp, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1);
2805 m_currentBlock->setSuccessors(isEqual, notEqual);
2806
2807 append(notEqual, branchOp, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1);
2808 notEqual->setSuccessors(isLessThan, notLessThan);
2809
2810 append(notLessThan, branchOp, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1);
2811 notLessThan->setSuccessors(isGreaterThan, isNaN);
2812
2813 auto andOp = floatType == F32 ? AndFloat : AndDouble;
2814 auto orOp = floatType == F32 ? OrFloat : OrDouble;
2815 append(isEqual, minOrMax == MinOrMax::Max ? andOp : orOp, arg0, arg1, result);
2816 append(isEqual, Jump);
2817 isEqual->setSuccessors(continuation);
2818
2819 auto isLessThanResult = minOrMax == MinOrMax::Max ? arg1 : arg0;
2820 append(isLessThan, moveOpForValueType(floatType), isLessThanResult, result);
2821 append(isLessThan, Jump);
2822 isLessThan->setSuccessors(continuation);
2823
2824 auto isGreaterThanResult = minOrMax == MinOrMax::Max ? arg0 : arg1;
2825 append(isGreaterThan, moveOpForValueType(floatType), isGreaterThanResult, result);
2826 append(isGreaterThan, Jump);
2827 isGreaterThan->setSuccessors(continuation);
2828
2829 auto addOp = floatType == F32 ? AddFloat : AddDouble;
2830 append(isNaN, addOp, arg0, arg1, result);
2831 append(isNaN, Jump);
2832 isNaN->setSuccessors(continuation);
2833
2834 m_currentBlock = continuation;
2835
2836 return { };
2837}
2838
2839template<> auto AirIRGenerator::addOp<OpType::F32Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2840{
2841 return addFloatingPointMinOrMax(F32, MinOrMax::Max, arg0, arg1, result);
2842}
2843
2844template<> auto AirIRGenerator::addOp<OpType::F64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2845{
2846 return addFloatingPointBinOp(Type::F64, MulDouble, arg0, arg1, result);
2847}
2848
2849template<> auto AirIRGenerator::addOp<OpType::F32Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2850{
2851 return addFloatingPointBinOp(Type::F32, DivFloat, arg0, arg1, result);
2852}
2853
2854template<> auto AirIRGenerator::addOp<OpType::I32Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2855{
2856 result = g32();
2857 append(CountLeadingZeros32, arg0, result);
2858 return { };
2859}
2860
2861template<> auto AirIRGenerator::addOp<OpType::F32Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2862{
2863 // FIXME: We can have better codegen here for the imms and two operand forms on x86
2864 // https://bugs.webkit.org/show_bug.cgi?id=193999
2865 result = f32();
2866 auto temp1 = g32();
2867 auto sign = g32();
2868 auto value = g32();
2869
2870 // FIXME: Try to use Imm where possible:
2871 // https://bugs.webkit.org/show_bug.cgi?id=193999
2872 append(MoveFloatTo32, arg1, temp1);
2873 append(Move, Arg::bigImm(0x80000000), sign);
2874 append(And32, temp1, sign, sign);
2875
2876 append(MoveDoubleTo64, arg0, temp1);
2877 append(Move, Arg::bigImm(0x7fffffff), value);
2878 append(And32, temp1, value, value);
2879
2880 append(Or32, sign, value, value);
2881 append(Move32ToFloat, value, result);
2882
2883 return { };
2884}
2885
2886template<> auto AirIRGenerator::addOp<OpType::F64ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2887{
2888 result = f64();
2889 auto temp = g64();
2890 append(Move32, arg0, temp);
2891 append(ConvertInt64ToDouble, temp, result);
2892 return { };
2893}
2894
2895template<> auto AirIRGenerator::addOp<OpType::F32ReinterpretI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2896{
2897 result = f32();
2898 append(Move32ToFloat, arg0, result);
2899 return { };
2900}
2901
2902template<> auto AirIRGenerator::addOp<OpType::I64And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2903{
2904 result = g64();
2905 append(And64, arg0, arg1, result);
2906 return { };
2907}
2908
2909template<> auto AirIRGenerator::addOp<OpType::F32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2910{
2911 result = g32();
2912 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
2913 return { };
2914}
2915
2916template<> auto AirIRGenerator::addOp<OpType::F64Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2917{
2918 result = g32();
2919 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
2920 return { };
2921}
2922
2923template<> auto AirIRGenerator::addOp<OpType::F32Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2924{
2925 result = f32();
2926 append(SqrtFloat, arg0, result);
2927 return { };
2928}
2929
2930template<> auto AirIRGenerator::addOp<OpType::F64Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2931{
2932 result = g32();
2933 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
2934 return { };
2935}
2936
2937template<> auto AirIRGenerator::addOp<OpType::I64GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2938{
2939 result = g32();
2940 append(Compare64, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
2941 return { };
2942}
2943
2944template<> auto AirIRGenerator::addOp<OpType::I64GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2945{
2946 result = g32();
2947 append(Compare64, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
2948 return { };
2949}
2950
2951template<> auto AirIRGenerator::addOp<OpType::I64Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2952{
2953 result = g32();
2954 append(Test64, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
2955 return { };
2956}
2957
2958template<> auto AirIRGenerator::addOp<OpType::F64Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2959{
2960 return addFloatingPointBinOp(Type::F64, DivDouble, arg0, arg1, result);
2961}
2962
2963template<> auto AirIRGenerator::addOp<OpType::F32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2964{
2965 result = f32();
2966 append(AddFloat, arg0, arg1, result);
2967 return { };
2968}
2969
2970template<> auto AirIRGenerator::addOp<OpType::I64Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2971{
2972 result = g64();
2973 append(Or64, arg0, arg1, result);
2974 return { };
2975}
2976
2977template<> auto AirIRGenerator::addOp<OpType::I32LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2978{
2979 result = g32();
2980 append(Compare32, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
2981 return { };
2982}
2983
2984template<> auto AirIRGenerator::addOp<OpType::I32LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2985{
2986 result = g32();
2987 append(Compare32, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
2988 return { };
2989}
2990
2991template<> auto AirIRGenerator::addOp<OpType::I64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2992{
2993 result = g32();
2994 append(Compare64, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
2995 return { };
2996}
2997
2998template<> auto AirIRGenerator::addOp<OpType::I64Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2999{
3000 result = g64();
3001 append(CountLeadingZeros64, arg0, result);
3002 return { };
3003}
3004
3005template<> auto AirIRGenerator::addOp<OpType::F32Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3006{
3007 result = f32();
3008 if (isValidForm(NegateFloat, Arg::Tmp, Arg::Tmp))
3009 append(NegateFloat, arg0, result);
3010 else {
3011 auto constant = addConstant(Type::I32, bitwise_cast<uint32_t>(static_cast<float>(-0.0)));
3012 auto temp = g32();
3013 append(MoveFloatTo32, arg0, temp);
3014 append(Xor32, constant, temp);
3015 append(Move32ToFloat, temp, result);
3016 }
3017 return { };
3018}
3019
3020template<> auto AirIRGenerator::addOp<OpType::I32And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3021{
3022 result = g32();
3023 append(And32, arg0, arg1, result);
3024 return { };
3025}
3026
3027template<> auto AirIRGenerator::addOp<OpType::I32LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3028{
3029 result = g32();
3030 append(Compare32, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3031 return { };
3032}
3033
3034template<> auto AirIRGenerator::addOp<OpType::I64Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3035{
3036 return addShift(Type::I64, RotateRight64, arg0, arg1, result);
3037}
3038
3039template<> auto AirIRGenerator::addOp<OpType::F64Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3040{
3041 return addFloatingPointAbs(AbsDouble, arg0, result);
3042}
3043
3044template<> auto AirIRGenerator::addOp<OpType::I32LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3045{
3046 result = g32();
3047 append(Compare32, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3048 return { };
3049}
3050
3051template<> auto AirIRGenerator::addOp<OpType::I32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3052{
3053 result = g32();
3054 append(Compare32, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3055 return { };
3056}
3057
3058template<> auto AirIRGenerator::addOp<OpType::F64Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3059{
3060 // FIXME: We can have better codegen here for the imms and two operand forms on x86
3061 // https://bugs.webkit.org/show_bug.cgi?id=193999
3062 result = f64();
3063 auto temp1 = g64();
3064 auto sign = g64();
3065 auto value = g64();
3066
3067 append(MoveDoubleTo64, arg1, temp1);
3068 append(Move, Arg::bigImm(0x8000000000000000), sign);
3069 append(And64, temp1, sign, sign);
3070
3071 append(MoveDoubleTo64, arg0, temp1);
3072 append(Move, Arg::bigImm(0x7fffffffffffffff), value);
3073 append(And64, temp1, value, value);
3074
3075 append(Or64, sign, value, value);
3076 append(Move64ToDouble, value, result);
3077
3078 return { };
3079}
3080
3081template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3082{
3083 result = f32();
3084 append(ConvertInt64ToFloat, arg0, result);
3085 return { };
3086}
3087
3088template<> auto AirIRGenerator::addOp<OpType::I64Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3089{
3090 if (isARM64()) {
3091 // ARM64 doesn't have a rotate left.
3092 auto newShift = g64();
3093 append(Move, arg1, newShift);
3094 append(Neg64, newShift);
3095 return addShift(Type::I64, RotateRight64, arg0, newShift, result);
3096 } else
3097 return addShift(Type::I64, RotateLeft64, arg0, arg1, result);
3098}
3099
3100template<> auto AirIRGenerator::addOp<OpType::F32Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3101{
3102 result = g32();
3103 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
3104 return { };
3105}
3106
3107template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3108{
3109 result = f64();
3110 append(ConvertInt32ToDouble, arg0, result);
3111 return { };
3112}
3113
3114template<> auto AirIRGenerator::addOp<OpType::F64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3115{
3116 result = g32();
3117 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3118 return { };
3119}
3120
3121template<> auto AirIRGenerator::addOp<OpType::F32Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3122{
3123 result = g32();
3124 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
3125 return { };
3126}
3127
3128template<> auto AirIRGenerator::addOp<OpType::F32Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3129{
3130 result = g32();
3131 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3132 return { };
3133}
3134
3135template<> auto AirIRGenerator::addOp<OpType::I32ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3136{
3137 return addShift(Type::I32, Urshift32, arg0, arg1, result);
3138}
3139
3140template<> auto AirIRGenerator::addOp<OpType::F32ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3141{
3142 result = f32();
3143 auto temp = g64();
3144 append(Move32, arg0, temp);
3145 append(ConvertInt64ToFloat, temp, result);
3146 return { };
3147}
3148
3149template<> auto AirIRGenerator::addOp<OpType::I32ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3150{
3151 return addShift(Type::I32, Rshift32, arg0, arg1, result);
3152}
3153
3154template<> auto AirIRGenerator::addOp<OpType::I32GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3155{
3156 result = g32();
3157 append(Compare32, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3158 return { };
3159}
3160
3161template<> auto AirIRGenerator::addOp<OpType::F64Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3162{
3163 result = f64();
3164 append(CeilDouble, arg0, result);
3165 return { };
3166}
3167
3168template<> auto AirIRGenerator::addOp<OpType::I32GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3169{
3170 result = g32();
3171 append(Compare32, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3172 return { };
3173}
3174
3175template<> auto AirIRGenerator::addOp<OpType::I32Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3176{
3177 return addShift(Type::I32, Lshift32, arg0, arg1, result);
3178}
3179
3180template<> auto AirIRGenerator::addOp<OpType::F64Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3181{
3182 result = f64();
3183 append(FloorDouble, arg0, result);
3184 return { };
3185}
3186
3187template<> auto AirIRGenerator::addOp<OpType::I32Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3188{
3189 result = g32();
3190 append(Xor32, arg0, arg1, result);
3191 return { };
3192}
3193
3194template<> auto AirIRGenerator::addOp<OpType::F32Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3195{
3196 return addFloatingPointAbs(AbsFloat, arg0, result);
3197}
3198
3199template<> auto AirIRGenerator::addOp<OpType::F64Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3200{
3201 return addFloatingPointMinOrMax(F64, MinOrMax::Min, arg0, arg1, result);
3202}
3203
3204template<> auto AirIRGenerator::addOp<OpType::F32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3205{
3206 result = f32();
3207 append(MulFloat, arg0, arg1, result);
3208 return { };
3209}
3210
3211template<> auto AirIRGenerator::addOp<OpType::I64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3212{
3213 return addIntegerSub(Sub64, arg0, arg1, result);
3214}
3215
3216template<> auto AirIRGenerator::addOp<OpType::I32ReinterpretF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3217{
3218 result = g32();
3219 append(MoveFloatTo32, arg0, result);
3220 return { };
3221}
3222
3223template<> auto AirIRGenerator::addOp<OpType::I32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3224{
3225 result = g32();
3226 append(Add32, arg0, arg1, result);
3227 return { };
3228}
3229
3230template<> auto AirIRGenerator::addOp<OpType::F64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3231{
3232 return addFloatingPointBinOp(Type::F64, SubDouble, arg0, arg1, result);
3233}
3234
3235template<> auto AirIRGenerator::addOp<OpType::I32Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3236{
3237 result = g32();
3238 append(Or32, arg0, arg1, result);
3239 return { };
3240}
3241
3242template<> auto AirIRGenerator::addOp<OpType::I64LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3243{
3244 result = g32();
3245 append(Compare64, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3246 return { };
3247}
3248
3249template<> auto AirIRGenerator::addOp<OpType::I64LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3250{
3251 result = g32();
3252 append(Compare64, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3253 return { };
3254}
3255
3256template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3257{
3258 result = f64();
3259 append(ConvertInt64ToDouble, arg0, result);
3260 return { };
3261}
3262
3263template<> auto AirIRGenerator::addOp<OpType::I64Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3264{
3265 result = g64();
3266 append(Xor64, arg0, arg1, result);
3267 return { };
3268}
3269
3270template<> auto AirIRGenerator::addOp<OpType::I64GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3271{
3272 result = g32();
3273 append(Compare64, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3274 return { };
3275}
3276
3277template<> auto AirIRGenerator::addOp<OpType::I64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3278{
3279 result = g64();
3280 append(Mul64, arg0, arg1, result);
3281 return { };
3282}
3283
3284template<> auto AirIRGenerator::addOp<OpType::F32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3285{
3286 result = f32();
3287 if (isValidForm(SubFloat, Arg::Tmp, Arg::Tmp, Arg::Tmp))
3288 append(SubFloat, arg0, arg1, result);
3289 else {
3290 RELEASE_ASSERT(isX86());
3291 append(MoveFloat, arg0, result);
3292 append(SubFloat, arg1, result);
3293 }
3294 return { };
3295}
3296
3297template<> auto AirIRGenerator::addOp<OpType::F64PromoteF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3298{
3299 result = f64();
3300 append(ConvertFloatToDouble, arg0, result);
3301 return { };
3302}
3303
3304template<> auto AirIRGenerator::addOp<OpType::F64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3305{
3306 result = f64();
3307 append(AddDouble, arg0, arg1, result);
3308 return { };
3309}
3310
3311template<> auto AirIRGenerator::addOp<OpType::I64GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3312{
3313 result = g32();
3314 append(Compare64, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3315 return { };
3316}
3317
3318template<> auto AirIRGenerator::addOp<OpType::I64ExtendUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3319{
3320 result = g64();
3321 append(Move32, arg0, result);
3322 return { };
3323}
3324
3325template<> auto AirIRGenerator::addOp<OpType::I32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3326{
3327 result = g32();
3328 RELEASE_ASSERT(arg0 && arg1);
3329 append(Compare32, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3330 return { };
3331}
3332
3333template<> auto AirIRGenerator::addOp<OpType::F64ReinterpretI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3334{
3335 result = f64();
3336 append(Move64ToDouble, arg0, result);
3337 return { };
3338}
3339
3340template<> auto AirIRGenerator::addOp<OpType::F32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3341{
3342 result = g32();
3343 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3344 return { };
3345}
3346
3347template<> auto AirIRGenerator::addOp<OpType::I64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3348{
3349 result = g32();
3350 append(Compare64, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3351 return { };
3352}
3353
3354template<> auto AirIRGenerator::addOp<OpType::F32Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3355{
3356 result = f32();
3357 append(FloorFloat, arg0, result);
3358 return { };
3359}
3360
3361template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3362{
3363 result = f32();
3364 append(ConvertInt32ToFloat, arg0, result);
3365 return { };
3366}
3367
3368template<> auto AirIRGenerator::addOp<OpType::I32Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3369{
3370 result = g32();
3371 append(Test32, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3372 return { };
3373}
3374
3375template<> auto AirIRGenerator::addOp<OpType::I64ReinterpretF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3376{
3377 result = g64();
3378 append(MoveDoubleTo64, arg0, result);
3379 return { };
3380}
3381
3382template<> auto AirIRGenerator::addOp<OpType::I64ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3383{
3384 return addShift(Type::I64, Rshift64, arg0, arg1, result);
3385}
3386
3387template<> auto AirIRGenerator::addOp<OpType::I64ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3388{
3389 return addShift(Type::I64, Urshift64, arg0, arg1, result);
3390}
3391
3392template<> auto AirIRGenerator::addOp<OpType::F64Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3393{
3394 result = f64();
3395 append(SqrtDouble, arg0, result);
3396 return { };
3397}
3398
3399template<> auto AirIRGenerator::addOp<OpType::I64Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3400{
3401 return addShift(Type::I64, Lshift64, arg0, arg1, result);
3402}
3403
3404template<> auto AirIRGenerator::addOp<OpType::F32Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3405{
3406 result = g32();
3407 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3408 return { };
3409}
3410
3411template<> auto AirIRGenerator::addOp<OpType::I32WrapI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3412{
3413 result = g32();
3414 append(Move32, arg0, result);
3415 return { };
3416}
3417
3418template<> auto AirIRGenerator::addOp<OpType::I32Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3419{
3420 if (isARM64()) {
3421 // ARM64 doesn't have a rotate left.
3422 auto newShift = g64();
3423 append(Move, arg1, newShift);
3424 append(Neg64, newShift);
3425 return addShift(Type::I32, RotateRight32, arg0, newShift, result);
3426 } else
3427 return addShift(Type::I32, RotateLeft32, arg0, arg1, result);
3428}
3429
3430template<> auto AirIRGenerator::addOp<OpType::I32Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3431{
3432 return addShift(Type::I32, RotateRight32, arg0, arg1, result);
3433}
3434
3435template<> auto AirIRGenerator::addOp<OpType::I32GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3436{
3437 result = g32();
3438 append(Compare32, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3439 return { };
3440}
3441
3442template<> auto AirIRGenerator::addOp<OpType::I64ExtendSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3443{
3444 result = g64();
3445 append(SignExtend32ToPtr, arg0, result);
3446 return { };
3447}
3448
3449template<> auto AirIRGenerator::addOp<OpType::I32GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3450{
3451 result = g32();
3452 append(Compare32, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3453 return { };
3454}
3455
3456template<> auto AirIRGenerator::addOp<OpType::F64Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3457{
3458 result = f64();
3459 if (isValidForm(NegateDouble, Arg::Tmp, Arg::Tmp))
3460 append(NegateDouble, arg0, result);
3461 else {
3462 auto constant = addConstant(Type::I64, bitwise_cast<uint64_t>(static_cast<double>(-0.0)));
3463 auto temp = g64();
3464 append(MoveDoubleTo64, arg0, temp);
3465 append(Xor64, constant, temp);
3466 append(Move64ToDouble, temp, result);
3467 }
3468 return { };
3469}
3470
3471template<> auto AirIRGenerator::addOp<OpType::F64Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3472{
3473 return addFloatingPointMinOrMax(F64, MinOrMax::Max, arg0, arg1, result);
3474}
3475
3476template<> auto AirIRGenerator::addOp<OpType::I64LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3477{
3478 result = g32();
3479 append(Compare64, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3480 return { };
3481}
3482
3483template<> auto AirIRGenerator::addOp<OpType::I64LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3484{
3485 result = g32();
3486 append(Compare64, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3487 return { };
3488}
3489
3490template<> auto AirIRGenerator::addOp<OpType::I64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3491{
3492 result = g64();
3493 append(Add64, arg0, arg1, result);
3494 return { };
3495}
3496
3497} } // namespace JSC::Wasm
3498
3499#endif // ENABLE(WEBASSEMBLY)
3500