1 // <memory_resource> -*- C++ -*-
3 // Copyright (C) 2018-2021 Free Software Foundation, Inc.
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
25 /** @file include/memory_resource
26 * This is a Standard C++ Library header.
29 #ifndef _GLIBCXX_MEMORY_RESOURCE
30 #define _GLIBCXX_MEMORY_RESOURCE 1
32 #pragma GCC system_header
34 #if __cplusplus >= 201703L
36 #include <vector> // vector
37 #include <cstddef> // size_t, max_align_t, byte
38 #include <shared_mutex> // shared_mutex
39 #include <bits/align.h> // align
40 #include <bits/functexcept.h> // __throw_bad_array_new_length
41 #include <bits/uses_allocator.h> // __use_alloc
42 #include <bits/uses_allocator_args.h> // uninitialized_construct_using_alloc
43 #include <ext/numeric_traits.h>
44 #include <debug/assertions.h>
46 #if ! __cpp_lib_make_obj_using_allocator
47 # include <utility> // pair, index_sequence
48 # include <tuple> // tuple, forward_as_tuple
51 namespace std _GLIBCXX_VISIBILITY(default)
53 _GLIBCXX_BEGIN_NAMESPACE_VERSION
56 #ifdef _GLIBCXX_HAS_GTHREADS
57 // Header and all contents are present.
58 # define __cpp_lib_memory_resource 201603L
60 // The pmr::synchronized_pool_resource type is missing.
61 # define __cpp_lib_memory_resource 1
64 class memory_resource;
66 #if __cplusplus == 201703L
67 template<typename _Tp>
68 class polymorphic_allocator;
70 # define __cpp_lib_polymorphic_allocator 201902L
71 template<typename _Tp = std::byte>
72 class polymorphic_allocator;
75 // Global memory resources
76 memory_resource* new_delete_resource() noexcept;
77 memory_resource* null_memory_resource() noexcept;
78 memory_resource* set_default_resource(memory_resource* __r) noexcept;
79 memory_resource* get_default_resource() noexcept
80 __attribute__((__returns_nonnull__));
82 // Pool resource classes
84 #ifdef _GLIBCXX_HAS_GTHREADS
85 class synchronized_pool_resource;
87 class unsynchronized_pool_resource;
88 class monotonic_buffer_resource;
90 /// Class memory_resource
93 static constexpr size_t _S_max_align = alignof(max_align_t);
96 memory_resource() = default;
97 memory_resource(const memory_resource&) = default;
98 virtual ~memory_resource(); // key function
100 memory_resource& operator=(const memory_resource&) = default;
104 allocate(size_t __bytes, size_t __alignment = _S_max_align)
105 __attribute__((__returns_nonnull__,__alloc_size__(2),__alloc_align__(3)))
106 { return do_allocate(__bytes, __alignment); }
109 deallocate(void* __p, size_t __bytes, size_t __alignment = _S_max_align)
110 __attribute__((__nonnull__))
111 { return do_deallocate(__p, __bytes, __alignment); }
114 is_equal(const memory_resource& __other) const noexcept
115 { return do_is_equal(__other); }
119 do_allocate(size_t __bytes, size_t __alignment) = 0;
122 do_deallocate(void* __p, size_t __bytes, size_t __alignment) = 0;
125 do_is_equal(const memory_resource& __other) const noexcept = 0;
129 operator==(const memory_resource& __a, const memory_resource& __b) noexcept
130 { return &__a == &__b || __a.is_equal(__b); }
132 #if __cpp_impl_three_way_comparison < 201907L
134 operator!=(const memory_resource& __a, const memory_resource& __b) noexcept
135 { return !(__a == __b); }
138 // C++17 23.12.3 Class template polymorphic_allocator
139 template<typename _Tp>
140 class polymorphic_allocator
142 // _GLIBCXX_RESOLVE_LIB_DEFECTS
143 // 2975. Missing case for pair construction in polymorphic allocators
144 template<typename _Up>
145 struct __not_pair { using type = void; };
147 template<typename _Up1, typename _Up2>
148 struct __not_pair<pair<_Up1, _Up2>> { };
151 using value_type = _Tp;
153 polymorphic_allocator() noexcept
154 : _M_resource(get_default_resource())
157 polymorphic_allocator(memory_resource* __r) noexcept
158 __attribute__((__nonnull__))
160 { _GLIBCXX_DEBUG_ASSERT(__r); }
162 polymorphic_allocator(const polymorphic_allocator& __other) = default;
164 template<typename _Up>
165 polymorphic_allocator(const polymorphic_allocator<_Up>& __x) noexcept
166 : _M_resource(__x.resource())
169 polymorphic_allocator&
170 operator=(const polymorphic_allocator&) = delete;
175 __attribute__((__returns_nonnull__))
177 if ((__gnu_cxx::__int_traits<size_t>::__max / sizeof(_Tp)) < __n)
178 std::__throw_bad_array_new_length();
179 return static_cast<_Tp*>(_M_resource->allocate(__n * sizeof(_Tp),
184 deallocate(_Tp* __p, size_t __n) noexcept
185 __attribute__((__nonnull__))
186 { _M_resource->deallocate(__p, __n * sizeof(_Tp), alignof(_Tp)); }
188 #if __cplusplus > 201703L
190 allocate_bytes(size_t __nbytes,
191 size_t __alignment = alignof(max_align_t))
192 { return _M_resource->allocate(__nbytes, __alignment); }
195 deallocate_bytes(void* __p, size_t __nbytes,
196 size_t __alignment = alignof(max_align_t))
197 { _M_resource->deallocate(__p, __nbytes, __alignment); }
199 template<typename _Up>
201 allocate_object(size_t __n = 1)
203 if ((__gnu_cxx::__int_traits<size_t>::__max / sizeof(_Up)) < __n)
204 std::__throw_bad_array_new_length();
205 return static_cast<_Up*>(allocate_bytes(__n * sizeof(_Up),
209 template<typename _Up>
211 deallocate_object(_Up* __p, size_t __n = 1)
212 { deallocate_bytes(__p, __n * sizeof(_Up), alignof(_Up)); }
214 template<typename _Up, typename... _CtorArgs>
216 new_object(_CtorArgs&&... __ctor_args)
218 _Up* __p = allocate_object<_Up>();
221 construct(__p, std::forward<_CtorArgs>(__ctor_args)...);
225 deallocate_object(__p);
226 __throw_exception_again;
231 template<typename _Up>
233 delete_object(_Up* __p)
236 deallocate_object(__p);
240 #if ! __cpp_lib_make_obj_using_allocator
241 template<typename _Tp1, typename... _Args>
242 __attribute__((__nonnull__))
243 typename __not_pair<_Tp1>::type
244 construct(_Tp1* __p, _Args&&... __args)
246 // _GLIBCXX_RESOLVE_LIB_DEFECTS
247 // 2969. polymorphic_allocator::construct() shouldn't pass resource()
249 = std::__uses_alloc_t<_Tp1, polymorphic_allocator, _Args...>;
250 if constexpr (is_base_of_v<__uses_alloc0, __use_tag>)
251 ::new(__p) _Tp1(std::forward<_Args>(__args)...);
252 else if constexpr (is_base_of_v<__uses_alloc1_, __use_tag>)
253 ::new(__p) _Tp1(allocator_arg, *this,
254 std::forward<_Args>(__args)...);
256 ::new(__p) _Tp1(std::forward<_Args>(__args)..., *this);
259 template<typename _Tp1, typename _Tp2,
260 typename... _Args1, typename... _Args2>
261 __attribute__((__nonnull__))
263 construct(pair<_Tp1, _Tp2>* __p, piecewise_construct_t,
264 tuple<_Args1...> __x, tuple<_Args2...> __y)
267 __use_alloc<_Tp1, polymorphic_allocator, _Args1...>(*this);
269 __use_alloc<_Tp2, polymorphic_allocator, _Args2...>(*this);
270 index_sequence_for<_Args1...> __x_i;
271 index_sequence_for<_Args2...> __y_i;
273 ::new(__p) pair<_Tp1, _Tp2>(piecewise_construct,
274 _S_construct_p(__x_tag, __x_i, __x),
275 _S_construct_p(__y_tag, __y_i, __y));
278 template<typename _Tp1, typename _Tp2>
279 __attribute__((__nonnull__))
281 construct(pair<_Tp1, _Tp2>* __p)
282 { this->construct(__p, piecewise_construct, tuple<>(), tuple<>()); }
284 template<typename _Tp1, typename _Tp2, typename _Up, typename _Vp>
285 __attribute__((__nonnull__))
287 construct(pair<_Tp1, _Tp2>* __p, _Up&& __x, _Vp&& __y)
289 this->construct(__p, piecewise_construct,
290 std::forward_as_tuple(std::forward<_Up>(__x)),
291 std::forward_as_tuple(std::forward<_Vp>(__y)));
294 template <typename _Tp1, typename _Tp2, typename _Up, typename _Vp>
295 __attribute__((__nonnull__))
297 construct(pair<_Tp1, _Tp2>* __p, const std::pair<_Up, _Vp>& __pr)
299 this->construct(__p, piecewise_construct,
300 std::forward_as_tuple(__pr.first),
301 std::forward_as_tuple(__pr.second));
304 template<typename _Tp1, typename _Tp2, typename _Up, typename _Vp>
305 __attribute__((__nonnull__))
307 construct(pair<_Tp1, _Tp2>* __p, pair<_Up, _Vp>&& __pr)
309 this->construct(__p, piecewise_construct,
310 std::forward_as_tuple(std::forward<_Up>(__pr.first)),
311 std::forward_as_tuple(std::forward<_Vp>(__pr.second)));
313 #else // make_obj_using_allocator
314 template<typename _Tp1, typename... _Args>
315 __attribute__((__nonnull__))
317 construct(_Tp1* __p, _Args&&... __args)
319 std::uninitialized_construct_using_allocator(__p, *this,
320 std::forward<_Args>(__args)...);
324 template<typename _Up>
325 __attribute__((__nonnull__))
330 polymorphic_allocator
331 select_on_container_copy_construction() const noexcept
332 { return polymorphic_allocator(); }
335 resource() const noexcept
336 __attribute__((__returns_nonnull__))
337 { return _M_resource; }
340 using __uses_alloc1_ = __uses_alloc1<polymorphic_allocator>;
341 using __uses_alloc2_ = __uses_alloc2<polymorphic_allocator>;
343 #if ! __cpp_lib_make_obj_using_allocator
344 template<typename _Ind, typename... _Args>
345 static tuple<_Args&&...>
346 _S_construct_p(__uses_alloc0, _Ind, tuple<_Args...>& __t)
347 { return std::move(__t); }
349 template<size_t... _Ind, typename... _Args>
350 static tuple<allocator_arg_t, polymorphic_allocator, _Args&&...>
351 _S_construct_p(__uses_alloc1_ __ua, index_sequence<_Ind...>,
352 tuple<_Args...>& __t)
355 allocator_arg, *__ua._M_a, std::get<_Ind>(std::move(__t))...
359 template<size_t... _Ind, typename... _Args>
360 static tuple<_Args&&..., polymorphic_allocator>
361 _S_construct_p(__uses_alloc2_ __ua, index_sequence<_Ind...>,
362 tuple<_Args...>& __t)
363 { return { std::get<_Ind>(std::move(__t))..., *__ua._M_a }; }
366 memory_resource* _M_resource;
369 template<typename _Tp1, typename _Tp2>
371 operator==(const polymorphic_allocator<_Tp1>& __a,
372 const polymorphic_allocator<_Tp2>& __b) noexcept
373 { return *__a.resource() == *__b.resource(); }
375 #if __cpp_impl_three_way_comparison < 201907L
376 template<typename _Tp1, typename _Tp2>
378 operator!=(const polymorphic_allocator<_Tp1>& __a,
379 const polymorphic_allocator<_Tp2>& __b) noexcept
380 { return !(__a == __b); }
383 /// Parameters for tuning a pool resource's behaviour.
386 /** @brief Upper limit on number of blocks in a chunk.
388 * A lower value prevents allocating huge chunks that could remain mostly
389 * unused, but means pools will need to replenished more frequently.
391 size_t max_blocks_per_chunk = 0;
393 /* @brief Largest block size (in bytes) that should be served from pools.
395 * Larger allocations will be served directly by the upstream resource,
396 * not from one of the pools managed by the pool resource.
398 size_t largest_required_pool_block = 0;
401 // Common implementation details for un-/synchronized pool resources.
402 class __pool_resource
404 friend class synchronized_pool_resource;
405 friend class unsynchronized_pool_resource;
407 __pool_resource(const pool_options& __opts, memory_resource* __upstream);
411 __pool_resource(const __pool_resource&) = delete;
412 __pool_resource& operator=(const __pool_resource&) = delete;
414 // Allocate a large unpooled block.
416 allocate(size_t __bytes, size_t __alignment);
418 // Deallocate a large unpooled block.
420 deallocate(void* __p, size_t __bytes, size_t __alignment);
423 // Deallocate unpooled memory.
424 void release() noexcept;
426 memory_resource* resource() const noexcept
427 { return _M_unpooled.get_allocator().resource(); }
431 _Pool* _M_alloc_pools();
433 const pool_options _M_opts;
436 // Collection of blocks too big for any pool, sorted by address.
437 // This also stores the only copy of the upstream memory resource pointer.
438 _GLIBCXX_STD_C::pmr::vector<_BigBlock> _M_unpooled;
443 #ifdef _GLIBCXX_HAS_GTHREADS
444 /// A thread-safe memory resource that manages pools of fixed-size blocks.
445 class synchronized_pool_resource : public memory_resource
448 synchronized_pool_resource(const pool_options& __opts,
449 memory_resource* __upstream)
450 __attribute__((__nonnull__));
452 synchronized_pool_resource()
453 : synchronized_pool_resource(pool_options(), get_default_resource())
457 synchronized_pool_resource(memory_resource* __upstream)
458 __attribute__((__nonnull__))
459 : synchronized_pool_resource(pool_options(), __upstream)
463 synchronized_pool_resource(const pool_options& __opts)
464 : synchronized_pool_resource(__opts, get_default_resource()) { }
466 synchronized_pool_resource(const synchronized_pool_resource&) = delete;
468 virtual ~synchronized_pool_resource();
470 synchronized_pool_resource&
471 operator=(const synchronized_pool_resource&) = delete;
476 upstream_resource() const noexcept
477 __attribute__((__returns_nonnull__))
478 { return _M_impl.resource(); }
480 pool_options options() const noexcept { return _M_impl._M_opts; }
484 do_allocate(size_t __bytes, size_t __alignment) override;
487 do_deallocate(void* __p, size_t __bytes, size_t __alignment) override;
490 do_is_equal(const memory_resource& __other) const noexcept override
491 { return this == &__other; }
494 // Thread-specific pools (only public for access by implementation details)
498 _TPools* _M_alloc_tpools(lock_guard<shared_mutex>&);
499 _TPools* _M_alloc_shared_tpools(lock_guard<shared_mutex>&);
500 auto _M_thread_specific_pools() noexcept;
502 __pool_resource _M_impl;
503 __gthread_key_t _M_key;
504 // Linked list of thread-specific pools. All threads share _M_tpools[0].
505 _TPools* _M_tpools = nullptr;
506 mutable shared_mutex _M_mx;
510 /// A non-thread-safe memory resource that manages pools of fixed-size blocks.
511 class unsynchronized_pool_resource : public memory_resource
514 [[__gnu__::__nonnull__]]
515 unsynchronized_pool_resource(const pool_options& __opts,
516 memory_resource* __upstream);
518 unsynchronized_pool_resource()
519 : unsynchronized_pool_resource(pool_options(), get_default_resource())
522 [[__gnu__::__nonnull__]]
524 unsynchronized_pool_resource(memory_resource* __upstream)
525 : unsynchronized_pool_resource(pool_options(), __upstream)
529 unsynchronized_pool_resource(const pool_options& __opts)
530 : unsynchronized_pool_resource(__opts, get_default_resource()) { }
532 unsynchronized_pool_resource(const unsynchronized_pool_resource&) = delete;
534 virtual ~unsynchronized_pool_resource();
536 unsynchronized_pool_resource&
537 operator=(const unsynchronized_pool_resource&) = delete;
541 [[__gnu__::__returns_nonnull__]]
543 upstream_resource() const noexcept
544 { return _M_impl.resource(); }
546 pool_options options() const noexcept { return _M_impl._M_opts; }
550 do_allocate(size_t __bytes, size_t __alignment) override;
553 do_deallocate(void* __p, size_t __bytes, size_t __alignment) override;
556 do_is_equal(const memory_resource& __other) const noexcept override
557 { return this == &__other; }
560 using _Pool = __pool_resource::_Pool;
562 auto _M_find_pool(size_t) noexcept;
564 __pool_resource _M_impl;
565 _Pool* _M_pools = nullptr;
568 class monotonic_buffer_resource : public memory_resource
572 monotonic_buffer_resource(memory_resource* __upstream) noexcept
573 __attribute__((__nonnull__))
574 : _M_upstream(__upstream)
575 { _GLIBCXX_DEBUG_ASSERT(__upstream != nullptr); }
577 monotonic_buffer_resource(size_t __initial_size,
578 memory_resource* __upstream) noexcept
579 __attribute__((__nonnull__))
580 : _M_next_bufsiz(__initial_size),
581 _M_upstream(__upstream)
583 _GLIBCXX_DEBUG_ASSERT(__upstream != nullptr);
584 _GLIBCXX_DEBUG_ASSERT(__initial_size > 0);
587 monotonic_buffer_resource(void* __buffer, size_t __buffer_size,
588 memory_resource* __upstream) noexcept
589 __attribute__((__nonnull__(4)))
590 : _M_current_buf(__buffer), _M_avail(__buffer_size),
591 _M_next_bufsiz(_S_next_bufsize(__buffer_size)),
592 _M_upstream(__upstream),
593 _M_orig_buf(__buffer), _M_orig_size(__buffer_size)
595 _GLIBCXX_DEBUG_ASSERT(__upstream != nullptr);
596 _GLIBCXX_DEBUG_ASSERT(__buffer != nullptr || __buffer_size == 0);
599 monotonic_buffer_resource() noexcept
600 : monotonic_buffer_resource(get_default_resource())
604 monotonic_buffer_resource(size_t __initial_size) noexcept
605 : monotonic_buffer_resource(__initial_size, get_default_resource())
608 monotonic_buffer_resource(void* __buffer, size_t __buffer_size) noexcept
609 : monotonic_buffer_resource(__buffer, __buffer_size, get_default_resource())
612 monotonic_buffer_resource(const monotonic_buffer_resource&) = delete;
614 virtual ~monotonic_buffer_resource(); // key function
616 monotonic_buffer_resource&
617 operator=(const monotonic_buffer_resource&) = delete;
623 _M_release_buffers();
625 // reset to initial state at contruction:
626 if ((_M_current_buf = _M_orig_buf))
628 _M_avail = _M_orig_size;
629 _M_next_bufsiz = _S_next_bufsize(_M_orig_size);
634 _M_next_bufsiz = _M_orig_size;
639 upstream_resource() const noexcept
640 __attribute__((__returns_nonnull__))
641 { return _M_upstream; }
645 do_allocate(size_t __bytes, size_t __alignment) override
647 if (__builtin_expect(__bytes == 0, false))
648 __bytes = 1; // Ensures we don't return the same pointer twice.
650 void* __p = std::align(__alignment, __bytes, _M_current_buf, _M_avail);
651 if (__builtin_expect(__p == nullptr, false))
653 _M_new_buffer(__bytes, __alignment);
654 __p = _M_current_buf;
656 _M_current_buf = (char*)_M_current_buf + __bytes;
662 do_deallocate(void*, size_t, size_t) override
666 do_is_equal(const memory_resource& __other) const noexcept override
667 { return this == &__other; }
670 // Update _M_current_buf and _M_avail to refer to a new buffer with
671 // at least the specified size and alignment, allocated from upstream.
673 _M_new_buffer(size_t __bytes, size_t __alignment);
675 // Deallocate all buffers obtained from upstream.
677 _M_release_buffers() noexcept;
680 _S_next_bufsize(size_t __buffer_size) noexcept
682 if (__builtin_expect(__buffer_size == 0, false))
684 return __buffer_size * _S_growth_factor;
687 static constexpr size_t _S_init_bufsize = 128 * sizeof(void*);
688 static constexpr float _S_growth_factor = 1.5;
690 void* _M_current_buf = nullptr;
692 size_t _M_next_bufsiz = _S_init_bufsize;
694 // Initial values set at construction and reused by release():
695 memory_resource* const _M_upstream;
696 void* const _M_orig_buf = nullptr;
697 size_t const _M_orig_size = _M_next_bufsiz;
700 _Chunk* _M_head = nullptr;
704 _GLIBCXX_END_NAMESPACE_VERSION
708 #endif // _GLIBCXX_MEMORY_RESOURCE