1/*
2 * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmOMGPlan.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "B3Compilation.h"
32#include "B3OpaqueByproducts.h"
33#include "JSCInlines.h"
34#include "LinkBuffer.h"
35#include "WasmB3IRGenerator.h"
36#include "WasmCallee.h"
37#include "WasmContext.h"
38#include "WasmInstance.h"
39#include "WasmMachineThreads.h"
40#include "WasmMemory.h"
41#include "WasmNameSection.h"
42#include "WasmSignatureInlines.h"
43#include "WasmValidate.h"
44#include "WasmWorklist.h"
45#include <wtf/DataLog.h>
46#include <wtf/Locker.h>
47#include <wtf/MonotonicTime.h>
48#include <wtf/StdLibExtras.h>
49#include <wtf/ThreadMessage.h>
50
51namespace JSC { namespace Wasm {
52
53namespace WasmOMGPlanInternal {
54static const bool verbose = false;
55}
56
57OMGPlan::OMGPlan(Context* context, Ref<Module>&& module, uint32_t functionIndex, MemoryMode mode, CompletionTask&& task)
58 : Base(context, makeRef(const_cast<ModuleInformation&>(module->moduleInformation())), WTFMove(task))
59 , m_module(WTFMove(module))
60 , m_codeBlock(*m_module->codeBlockFor(mode))
61 , m_functionIndex(functionIndex)
62{
63 setMode(mode);
64 ASSERT(m_codeBlock->runnable());
65 ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(m_mode));
66 dataLogLnIf(WasmOMGPlanInternal::verbose, "Starting OMG plan for ", functionIndex, " of module: ", RawPointer(&m_module.get()));
67}
68
69void OMGPlan::work(CompilationEffort)
70{
71 ASSERT(m_codeBlock->runnable());
72 ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(mode()));
73 const FunctionData& function = m_moduleInformation->functions[m_functionIndex];
74
75 const uint32_t functionIndexSpace = m_functionIndex + m_module->moduleInformation().importFunctionCount();
76 ASSERT(functionIndexSpace < m_module->moduleInformation().functionIndexSpaceSize());
77
78 SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[m_functionIndex];
79 const Signature& signature = SignatureInformation::get(signatureIndex);
80 ASSERT(validateFunction(function.data.data(), function.data.size(), signature, m_moduleInformation.get()));
81
82 Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
83 CompilationContext context;
84 auto parseAndCompileResult = parseAndCompile(context, function.data.data(), function.data.size(), signature, unlinkedCalls, m_moduleInformation.get(), m_mode, CompilationMode::OMGMode, m_functionIndex);
85
86 if (UNLIKELY(!parseAndCompileResult)) {
87 fail(holdLock(m_lock), makeString(parseAndCompileResult.error(), "when trying to tier up ", String::number(m_functionIndex)));
88 return;
89 }
90
91 Entrypoint omgEntrypoint;
92 LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, JITCompilationCanFail);
93 if (UNLIKELY(linkBuffer.didFailToAllocate())) {
94 Base::fail(holdLock(m_lock), makeString("Out of executable memory while tiering up function at index ", String::number(m_functionIndex)));
95 return;
96 }
97
98 omgEntrypoint.compilation = std::make_unique<B3::Compilation>(
99 FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "WebAssembly OMG function[%i] %s", m_functionIndex, signature.toString().ascii().data()),
100 WTFMove(context.wasmEntrypointByproducts));
101
102 omgEntrypoint.calleeSaveRegisters = WTFMove(parseAndCompileResult.value()->entrypoint.calleeSaveRegisters);
103
104 MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
105 {
106 ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(mode()));
107 Ref<Callee> callee = Callee::create(WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace));
108 MacroAssembler::repatchPointer(parseAndCompileResult.value()->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
109 ASSERT(!m_codeBlock->m_optimizedCallees[m_functionIndex]);
110 entrypoint = callee->entrypoint();
111
112 // We want to make sure we publish our callee at the same time as we link our callsites. This enables us to ensure we
113 // always call the fastest code. Any function linked after us will see our new code and the new callsites, which they
114 // will update. It's also ok if they publish their code before we reset the instruction caches because after we release
115 // the lock our code is ready to be published too.
116 LockHolder holder(m_codeBlock->m_lock);
117 m_codeBlock->m_optimizedCallees[m_functionIndex] = WTFMove(callee);
118
119 for (auto& call : unlinkedCalls) {
120 MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
121 if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
122 entrypoint = m_codeBlock->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
123 else
124 entrypoint = m_codeBlock->wasmEntrypointCalleeFromFunctionIndexSpace(call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
125
126 MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
127 }
128 unlinkedCalls = std::exchange(m_codeBlock->m_wasmToWasmCallsites[m_functionIndex], unlinkedCalls);
129 }
130
131 // It's important to make sure we do this before we make any of the code we just compiled visible. If we didn't, we could end up
132 // where we are tiering up some function A to A' and we repatch some function B to call A' instead of A. Another CPU could see
133 // the updates to B but still not have reset its cache of A', which would lead to all kinds of badness.
134 resetInstructionCacheOnAllThreads();
135 WTF::storeStoreFence(); // This probably isn't necessary but it's good to be paranoid.
136
137 m_codeBlock->m_wasmIndirectCallEntryPoints[m_functionIndex] = entrypoint;
138 {
139 LockHolder holder(m_codeBlock->m_lock);
140
141 auto repatchCalls = [&] (const Vector<UnlinkedWasmToWasmCall>& callsites) {
142 for (auto& call : callsites) {
143 dataLogLnIf(WasmOMGPlanInternal::verbose, "Considering repatching call at: ", RawPointer(call.callLocation.dataLocation()), " that targets ", call.functionIndexSpace);
144 if (call.functionIndexSpace == functionIndexSpace) {
145 dataLogLnIf(WasmOMGPlanInternal::verbose, "Repatching call at: ", RawPointer(call.callLocation.dataLocation()), " to ", RawPointer(entrypoint.executableAddress()));
146 MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
147 }
148 }
149
150 };
151
152 for (unsigned i = 0; i < m_codeBlock->m_wasmToWasmCallsites.size(); ++i) {
153 if (i != functionIndexSpace)
154 repatchCalls(m_codeBlock->m_wasmToWasmCallsites[i]);
155 }
156
157 // Make sure we repatch any recursive calls.
158 repatchCalls(unlinkedCalls);
159 }
160
161 dataLogLnIf(WasmOMGPlanInternal::verbose, "Finished with tier up count at: ", m_codeBlock->tierUpCount(m_functionIndex).count());
162 complete(holdLock(m_lock));
163}
164
165void OMGPlan::runForIndex(Instance* instance, uint32_t functionIndex)
166{
167 Wasm::CodeBlock& codeBlock = *instance->codeBlock();
168 ASSERT(instance->memory()->mode() == codeBlock.mode());
169
170 if (codeBlock.tierUpCount(functionIndex).shouldStartTierUp()) {
171 Ref<Plan> plan = adoptRef(*new OMGPlan(instance->context(), Ref<Wasm::Module>(instance->module()), functionIndex, codeBlock.mode(), Plan::dontFinalize()));
172 ensureWorklist().enqueue(plan.copyRef());
173 if (UNLIKELY(!Options::useConcurrentJIT()))
174 plan->waitForCompletion();
175 }
176}
177
178} } // namespace JSC::Wasm
179
180#endif // ENABLE(WEBASSEMBLY)
181