1/*
2 * Copyright (C) 2005-2018 Apple Inc. All rights reserved.
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public License
15 * along with this library; see the file COPYING.LIB. If not, write to
16 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 *
19 */
20
21#pragma once
22
23#include <new>
24#include <stdlib.h>
25#include <wtf/StdLibExtras.h>
26
27namespace WTF {
28
29#if !defined(NDEBUG)
30WTF_EXPORT_PRIVATE void fastSetMaxSingleAllocationSize(size_t);
31#endif
32
33class TryMallocReturnValue {
34public:
35 TryMallocReturnValue(void*);
36 TryMallocReturnValue(TryMallocReturnValue&&);
37 ~TryMallocReturnValue();
38 template<typename T> bool getValue(T*&) WARN_UNUSED_RETURN;
39private:
40 void operator=(TryMallocReturnValue&&) = delete;
41 mutable void* m_data;
42};
43
44WTF_EXPORT_PRIVATE bool isFastMallocEnabled();
45
46// These functions call CRASH() if an allocation fails.
47WTF_EXPORT_PRIVATE void* fastMalloc(size_t) RETURNS_NONNULL;
48WTF_EXPORT_PRIVATE void* fastZeroedMalloc(size_t) RETURNS_NONNULL;
49WTF_EXPORT_PRIVATE void* fastCalloc(size_t numElements, size_t elementSize) RETURNS_NONNULL;
50WTF_EXPORT_PRIVATE void* fastRealloc(void*, size_t) RETURNS_NONNULL;
51WTF_EXPORT_PRIVATE char* fastStrDup(const char*) RETURNS_NONNULL;
52
53WTF_EXPORT_PRIVATE TryMallocReturnValue tryFastMalloc(size_t);
54WTF_EXPORT_PRIVATE TryMallocReturnValue tryFastZeroedMalloc(size_t);
55WTF_EXPORT_PRIVATE TryMallocReturnValue tryFastCalloc(size_t numElements, size_t elementSize);
56WTF_EXPORT_PRIVATE TryMallocReturnValue tryFastRealloc(void*, size_t);
57
58WTF_EXPORT_PRIVATE void fastFree(void*);
59
60// Allocations from fastAlignedMalloc() must be freed using fastAlignedFree().
61WTF_EXPORT_PRIVATE void* fastAlignedMalloc(size_t alignment, size_t) RETURNS_NONNULL;
62WTF_EXPORT_PRIVATE void* tryFastAlignedMalloc(size_t alignment, size_t);
63WTF_EXPORT_PRIVATE void fastAlignedFree(void*);
64
65WTF_EXPORT_PRIVATE size_t fastMallocSize(const void*);
66
67// FIXME: This is non-helpful; fastMallocGoodSize will be removed soon.
68WTF_EXPORT_PRIVATE size_t fastMallocGoodSize(size_t);
69
70WTF_EXPORT_PRIVATE void releaseFastMallocFreeMemory();
71WTF_EXPORT_PRIVATE void releaseFastMallocFreeMemoryForThisThread();
72
73WTF_EXPORT_PRIVATE void fastCommitAlignedMemory(void*, size_t);
74WTF_EXPORT_PRIVATE void fastDecommitAlignedMemory(void*, size_t);
75
76WTF_EXPORT_PRIVATE void fastEnableMiniMode();
77
78struct FastMallocStatistics {
79 size_t reservedVMBytes;
80 size_t committedVMBytes;
81 size_t freeListBytes;
82};
83WTF_EXPORT_PRIVATE FastMallocStatistics fastMallocStatistics();
84
85// This defines a type which holds an unsigned integer and is the same
86// size as the minimally aligned memory allocation.
87typedef unsigned long long AllocAlignmentInteger;
88
89inline TryMallocReturnValue::TryMallocReturnValue(void* data)
90 : m_data(data)
91{
92}
93
94inline TryMallocReturnValue::TryMallocReturnValue(TryMallocReturnValue&& source)
95 : m_data(source.m_data)
96{
97 source.m_data = nullptr;
98}
99
100inline TryMallocReturnValue::~TryMallocReturnValue()
101{
102 ASSERT(!m_data);
103}
104
105template<typename T> inline bool TryMallocReturnValue::getValue(T*& data)
106{
107 data = static_cast<T*>(m_data);
108 m_data = nullptr;
109 return data;
110}
111
112// C++ STL allocator implementation. You can integrate fastMalloc into STL containers.
113// e.g. std::unordered_map<Key, Value, std::hash<Key>, std::equal_to<Key>, FastAllocator<std::pair<const Key, Value>>>.
114template<typename T>
115class FastAllocator {
116public:
117 using value_type = T;
118
119 FastAllocator() = default;
120
121 template<typename U> FastAllocator(const FastAllocator<U>&) { }
122
123 T* allocate(size_t count)
124 {
125 return reinterpret_cast<T*>(fastMalloc(sizeof(T) * count));
126 }
127
128 void deallocate(T* pointer, size_t)
129 {
130 fastFree(pointer);
131 }
132
133#if defined(__GLIBCXX__) && (!defined(_GLIBCXX_RELEASE) || _GLIBCXX_RELEASE < 6)
134 // This allocator also supports pre-C++11 STL allocator interface. This is a workaround for GCC < 6, which std::list
135 // does not support C++11 allocator. Note that _GLIBCXX_RELEASE is only defined after GCC 7 release. So currently
136 // this workaround is enabled in GCC 6 too.
137 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55409
138
139 using pointer = value_type*;
140 using const_pointer = typename std::pointer_traits<pointer>::template rebind<value_type const>;
141 using void_pointer = typename std::pointer_traits<pointer>::template rebind<void>;
142 using const_void_pointer = typename std::pointer_traits<pointer>::template rebind<const void>;
143
144 using reference = T&;
145 using const_reference = const T&;
146
147 using difference_type = typename std::pointer_traits<pointer>::difference_type;
148 using size_type = std::make_unsigned_t<difference_type>;
149
150 template <class U> struct rebind {
151 using other = FastAllocator<U>;
152 };
153
154 value_type* allocate(std::size_t count, const_void_pointer)
155 {
156 return allocate(count);
157 }
158
159 template <class U, class ...Args>
160 void construct(U* p, Args&& ...args)
161 {
162 new (const_cast<void*>(static_cast<const void*>(p))) U(std::forward<Args>(args)...);
163 }
164
165 template <class U>
166 void destroy(U* p)
167 {
168 p->~U();
169 }
170
171 std::size_t max_size() const
172 {
173 return std::numeric_limits<size_type>::max();
174 }
175
176 FastAllocator<T> select_on_container_copy_construction() const
177 {
178 return *this;
179 }
180
181 using propagate_on_container_copy_assignment = std::false_type;
182 using propagate_on_container_move_assignment = std::false_type;
183 using propagate_on_container_swap = std::false_type;
184 using is_always_equal = std::is_empty<FastAllocator>;
185#endif // defined(__GLIBCXX__) && (!defined(_GLIBCXX_RELEASE) || _GLIBCXX_RELEASE < 6)
186};
187
188template<typename T, typename U> inline bool operator==(const FastAllocator<T>&, const FastAllocator<U>&) { return true; }
189template<typename T, typename U> inline bool operator!=(const FastAllocator<T>&, const FastAllocator<U>&) { return false; }
190
191struct FastMalloc {
192 static void* malloc(size_t size) { return fastMalloc(size); }
193
194 static void* tryMalloc(size_t size)
195 {
196 auto result = tryFastMalloc(size);
197 void* realResult;
198 if (result.getValue(realResult))
199 return realResult;
200 return nullptr;
201 }
202
203 static void* realloc(void* p, size_t size) { return fastRealloc(p, size); }
204
205 static void* tryRealloc(void* p, size_t size)
206 {
207 auto result = tryFastRealloc(p, size);
208 void* realResult;
209 if (result.getValue(realResult))
210 return realResult;
211 return nullptr;
212 }
213
214 static void free(void* p) { fastFree(p); }
215};
216
217template<typename T>
218struct FastFree {
219 static_assert(std::is_trivially_destructible<T>::value, "");
220
221 void operator()(T* pointer) const
222 {
223 fastFree(const_cast<typename std::remove_cv<T>::type*>(pointer));
224 }
225};
226
227template<typename T>
228struct FastFree<T[]> {
229 static_assert(std::is_trivially_destructible<T>::value, "");
230
231 void operator()(T* pointer) const
232 {
233 fastFree(const_cast<typename std::remove_cv<T>::type*>(pointer));
234 }
235};
236
237} // namespace WTF
238
239#if !defined(NDEBUG)
240using WTF::fastSetMaxSingleAllocationSize;
241#endif
242
243using WTF::FastAllocator;
244using WTF::FastMalloc;
245using WTF::FastFree;
246using WTF::isFastMallocEnabled;
247using WTF::fastCalloc;
248using WTF::fastFree;
249using WTF::fastMalloc;
250using WTF::fastMallocGoodSize;
251using WTF::fastMallocSize;
252using WTF::fastRealloc;
253using WTF::fastStrDup;
254using WTF::fastZeroedMalloc;
255using WTF::tryFastAlignedMalloc;
256using WTF::tryFastCalloc;
257using WTF::tryFastMalloc;
258using WTF::tryFastZeroedMalloc;
259using WTF::fastAlignedMalloc;
260using WTF::fastAlignedFree;
261
262#if COMPILER(GCC_COMPATIBLE) && OS(DARWIN)
263#define WTF_PRIVATE_INLINE __private_extern__ inline __attribute__((always_inline))
264#elif COMPILER(GCC_COMPATIBLE)
265#define WTF_PRIVATE_INLINE inline __attribute__((always_inline))
266#elif COMPILER(MSVC)
267#define WTF_PRIVATE_INLINE __forceinline
268#else
269#define WTF_PRIVATE_INLINE inline
270#endif
271
272#define WTF_MAKE_FAST_ALLOCATED_IMPL \
273 void* operator new(size_t, void* p) { return p; } \
274 void* operator new[](size_t, void* p) { return p; } \
275 \
276 void* operator new(size_t size) \
277 { \
278 return ::WTF::fastMalloc(size); \
279 } \
280 \
281 void operator delete(void* p) \
282 { \
283 ::WTF::fastFree(p); \
284 } \
285 \
286 void* operator new[](size_t size) \
287 { \
288 return ::WTF::fastMalloc(size); \
289 } \
290 \
291 void operator delete[](void* p) \
292 { \
293 ::WTF::fastFree(p); \
294 } \
295 void* operator new(size_t, NotNullTag, void* location) \
296 { \
297 ASSERT(location); \
298 return location; \
299 } \
300
301#define WTF_MAKE_FAST_ALLOCATED \
302public: \
303 WTF_MAKE_FAST_ALLOCATED_IMPL \
304private: \
305typedef int __thisIsHereToForceASemicolonAfterThisMacro
306
307#define WTF_MAKE_STRUCT_FAST_ALLOCATED \
308 WTF_MAKE_FAST_ALLOCATED_IMPL \
309typedef int __thisIsHereToForceASemicolonAfterThisMacro
310