Actual source code: object_pool.hpp
1: #pragma once
3: #include <petsc/private/petscimpl.h>
4: #include <petsc/private/mempoison.h>
6: #include <petsc/private/cpp/register_finalize.hpp>
7: #include <petsc/private/cpp/utility.hpp>
8: #include <petsc/private/cpp/unordered_map.hpp>
9: #include <petsc/private/cpp/memory.hpp>
11: #include <cstddef> // std::size_t
12: #include <vector> // std::take_a_wild_guess
14: namespace Petsc
15: {
17: namespace memory
18: {
20: enum class align_val_t : std::size_t {
21: };
23: } // namespace memory
25: } // namespace Petsc
27: namespace std
28: {
30: template <>
31: struct hash<::Petsc::memory::align_val_t> {
32: #if PETSC_CPP_VERSION < 17
33: using argument_type = ::Petsc::memory::align_val_t;
34: using result_type = size_t;
35: #endif
37: constexpr size_t operator()(const ::Petsc::memory::align_val_t &x) const noexcept { return static_cast<size_t>(x); }
38: };
40: } // namespace std
42: namespace Petsc
43: {
45: namespace memory
46: {
48: // ==========================================================================================
49: // PoolAllocator
50: //
51: // A general purpose memory pool. It internally maintains an array of allocated memory regions
52: // and their sizes. Currently does not prune the allocated memory in any way.
53: // ==========================================================================================
55: class PoolAllocator : public RegisterFinalizeable<PoolAllocator> {
56: using base_type = RegisterFinalizeable<PoolAllocator>;
57: friend base_type;
59: public:
60: // define the size and alignment as separate types, this helps to disambiguate them at the
61: // callsite!
62: using size_type = std::size_t;
63: using align_type = align_val_t;
64: using pool_type = std::vector<std::pair<align_type, UnorderedMap<size_type, std::vector<void *>>>>;
66: PoolAllocator() noexcept = default;
67: PoolAllocator(PoolAllocator &&) noexcept = default;
68: PoolAllocator &operator=(PoolAllocator &&) noexcept = default;
70: // the pool carries raw memory and is not copyable
71: PoolAllocator(const PoolAllocator &) = delete;
72: PoolAllocator &operator=(const PoolAllocator &) = delete;
74: ~PoolAllocator() noexcept;
76: PetscErrorCode try_allocate(void **, size_type, align_type, bool *) noexcept;
77: PetscErrorCode allocate(void **, size_type, align_type, bool * = nullptr) noexcept;
78: PetscErrorCode deallocate(void **, size_type, align_type) noexcept;
80: static PetscErrorCode get_attributes(const void *, size_type *, align_type *) noexcept;
81: static PetscErrorCode unpoison(const void *, size_type *) noexcept;
82: static PetscErrorCode repoison(const void *, size_type) noexcept;
84: template <typename T>
85: PetscErrorCode for_each(T &&) noexcept;
87: class LockGuard {
88: public:
89: LockGuard() = delete;
91: private:
92: friend class PoolAllocator;
94: explicit LockGuard(PoolAllocator *pool) noexcept : pool_{pool} { ++pool_->locked_; }
96: struct PoolUnlocker {
97: void operator()(PoolAllocator *pool) const noexcept { --pool->locked_; }
98: };
100: std::unique_ptr<PoolAllocator, PoolUnlocker> pool_{};
101: };
103: LockGuard lock_guard() noexcept;
105: private:
106: pool_type pool_;
107: int locked_ = 0;
109: struct AllocationHeader;
111: PETSC_NODISCARD static constexpr size_type total_size_(size_type, align_type) noexcept;
113: static PetscErrorCode valid_alignment_(align_type) noexcept;
114: static PetscErrorCode extract_header_(void *, AllocationHeader **, bool = true) noexcept;
115: static PetscErrorCode allocate_ptr_(size_type, align_type, void **) noexcept;
117: static PetscErrorCode delete_ptr_(void **) noexcept;
119: PETSC_NODISCARD pool_type &pool() noexcept { return pool_; }
120: PETSC_NODISCARD const pool_type &pool() const noexcept { return pool_; }
122: PETSC_NODISCARD typename pool_type::iterator find_align_(align_type) noexcept;
123: PETSC_NODISCARD typename pool_type::const_iterator find_align_(align_type) const noexcept;
125: public:
126: PetscErrorCode clear_(size_type * = nullptr) noexcept;
127: PetscErrorCode finalize_() noexcept;
128: };
130: /*
131: PoolAllocator::for_each - Perform an action on every allocation in the currently in the pool
133: Input Parameter:
134: . callable - The callable used to perform the action, should accept a void *&.
136: Notes:
137: The callable may delete the pointer, but MUST set the pointer to nullptr in this case. If the
138: pointer is deleted, it is removed from the pool.
140: The pointers are walked in LIFO order from most recent first to least recent deallocation
141: last.
142: */
143: template <typename T>
144: inline PetscErrorCode PoolAllocator::for_each(T &&callable) noexcept
145: {
146: PetscFunctionBegin;
147: for (auto &&align_it : pool()) {
148: for (auto &&size_it : align_it.second) {
149: auto &&ptr_stack = size_it.second;
151: for (auto it = ptr_stack.rbegin(); it != ptr_stack.rend();) {
152: size_type size;
153: auto &&ptr = *it;
155: PetscCall(unpoison(ptr, &size));
156: PetscCall(callable(ptr));
157: if (ptr) {
158: PetscCall(repoison(ptr, size));
159: ++it;
160: } else {
161: // the callable has deleted the pointer, so we should remove it. it is a reverse
162: // iterator though so we:
163: //
164: // 1. std::next(it).base() -> convert to forward iterator
165: // 2. it = decltype(it){...} -> convert returned iterator back to reverse iterator
166: PetscCallCXX(it = decltype(it){ptr_stack.erase(std::next(it).base())});
167: }
168: }
169: }
170: }
171: PetscFunctionReturn(PETSC_SUCCESS);
172: }
174: // ==========================================================================================
175: // PoolAllocated
176: //
177: // A simple mixin to enable allocating a class from a Pool. That is, it provides a static
178: // operator new() and operator delete() member functions such that
179: //
180: // auto foo = new ClassDerivedFromPoolAllocated(args...);
181: //
182: // Allocates the new object from a pool and
183: //
184: // delete foo;
185: //
186: // Returns the memory to the pool.
187: // ==========================================================================================
189: class PoolAllocated {
190: public:
191: using allocator_type = PoolAllocator;
192: using size_type = typename allocator_type::size_type;
193: using align_type = typename allocator_type::align_type;
195: PETSC_NODISCARD static void *operator new(size_type) noexcept;
196: static void operator delete(void *) noexcept;
198: #if PETSC_CPP_VERSION >= 17
199: PETSC_NODISCARD static void *operator new(size_type, std::align_val_t) noexcept;
200: static void operator delete(void *, std::align_val_t) noexcept;
201: #endif
203: protected:
204: PETSC_NODISCARD static allocator_type &pool() noexcept;
206: private:
207: static allocator_type pool_;
208: };
210: } // namespace memory
212: // ==========================================================================================
213: // ConstructorInterface
214: //
215: // Provides a common interface for constructors and destructors for use with the object
216: // pools. Specifically, each interface may provide the following functions:
217: //
218: // construct_(T *):
219: // Given a pointer to an allocated object, construct the object in that memory. This may
220: // allocate memory *within* the object, but must not reallocate the pointer itself. Defaults to
221: // placement new.
222: //
223: // destroy_(T *):
224: // Given a pointer to an object, destroy the object completely. This should clean up the
225: // pointed-to object but not deallocate the pointer itself. Any resources not cleaned up by
226: // this function will be leaked. Defaults to calling the objects destructor.
227: //
228: // invalidate_(T *):
229: // Similar to destroy_(), but you are allowed to leave resources behind in the
230: // object. Essentially puts the object into a "zombie" state to be reused later. Defaults to
231: // calling destroy_().
232: //
233: // reset_():
234: // Revives a previously invalidated object. Should restore the object to a "factory new" state
235: // as-if it had just been newly allocated, but may take advantage of the fact that some
236: // resources need not be re-aquired from scratch. Defaults to calling construct_().
237: // ==========================================================================================
239: template <typename T, typename Derived>
240: class ConstructorInterface {
241: public:
242: using value_type = T;
244: template <typename... Args>
245: PetscErrorCode construct(value_type *ptr, Args &&...args) const noexcept
246: {
247: PetscFunctionBegin;
248: PetscCall(this->underlying().construct_(ptr, std::forward<Args>(args)...));
249: PetscFunctionReturn(PETSC_SUCCESS);
250: }
252: PetscErrorCode destroy(value_type *ptr) const noexcept
253: {
254: PetscFunctionBegin;
255: PetscCall(this->underlying().destroy_(ptr));
256: PetscFunctionReturn(PETSC_SUCCESS);
257: }
259: template <typename... Args>
260: PetscErrorCode reset(value_type *val, Args &&...args) const noexcept
261: {
262: PetscFunctionBegin;
263: PetscCall(this->underlying().reset_(val, std::forward<Args>(args)...));
264: PetscFunctionReturn(PETSC_SUCCESS);
265: }
267: PetscErrorCode invalidate(value_type *ptr) const noexcept
268: {
269: PetscFunctionBegin;
270: PetscCall(this->underlying().invalidate_(ptr));
271: PetscFunctionReturn(PETSC_SUCCESS);
272: }
274: protected:
275: template <typename... Args>
276: static PetscErrorCode construct_(value_type *ptr, Args &&...args) noexcept
277: {
278: PetscFunctionBegin;
279: PetscAssertPointer(ptr, 1);
280: PetscCallCXX(util::construct_at(ptr, std::forward<Args>(args)...));
281: PetscFunctionReturn(PETSC_SUCCESS);
282: }
284: static PetscErrorCode destroy_(value_type *ptr) noexcept
285: {
286: PetscFunctionBegin;
287: if (ptr) {
288: PetscAssertPointer(ptr, 1);
289: PetscCallCXX(util::destroy_at(ptr));
290: }
291: PetscFunctionReturn(PETSC_SUCCESS);
292: }
294: template <typename... Args>
295: PetscErrorCode reset_(value_type *val, Args &&...args) const noexcept
296: {
297: PetscFunctionBegin;
298: PetscCall(this->underlying().construct(val, std::forward<Args>(args)...));
299: PetscFunctionReturn(PETSC_SUCCESS);
300: }
302: PetscErrorCode invalidate_(value_type *ptr) const noexcept
303: {
304: PetscFunctionBegin;
305: PetscCall(this->underlying().destroy(ptr));
306: PetscFunctionReturn(PETSC_SUCCESS);
307: }
309: PETSC_NODISCARD Derived &underlying() noexcept { return static_cast<Derived &>(*this); }
310: PETSC_NODISCARD const Derived &underlying() const noexcept { return static_cast<const Derived &>(*this); }
311: };
313: template <typename T>
314: struct DefaultConstructor : ConstructorInterface<T, DefaultConstructor<T>> { };
316: // ==========================================================================================
317: // ObjectPool
318: //
319: // multi-purpose basic object-pool, useful for recirculating old "destroyed" objects. Registers
320: // all objects to be cleaned up on PetscFinalize()
321: // ==========================================================================================
323: template <typename T, typename Constructor = DefaultConstructor<T>>
324: class ObjectPool;
326: template <typename T, typename Constructor>
327: class ObjectPool : public RegisterFinalizeable<ObjectPool<T, Constructor>> {
328: using base_type = RegisterFinalizeable<ObjectPool<T, Constructor>>;
330: public:
331: using value_type = T;
332: using constructor_type = Constructor;
333: using allocator_type = memory::PoolAllocator;
335: ObjectPool() = default;
336: ObjectPool(ObjectPool &&) noexcept = default;
337: ObjectPool &operator=(ObjectPool &&) noexcept = default;
339: ~ObjectPool() noexcept;
341: template <typename... Args>
342: PetscErrorCode allocate(value_type **, Args &&...) noexcept;
343: PetscErrorCode deallocate(value_type **) noexcept;
345: PETSC_NODISCARD constructor_type &constructor() noexcept { return pair_.first(); }
346: PETSC_NODISCARD const constructor_type &constructor() const noexcept { return pair_.first(); }
347: PETSC_NODISCARD allocator_type &allocator() noexcept { return pair_.second(); }
348: PETSC_NODISCARD const allocator_type &allocator() const noexcept { return pair_.second(); }
350: private:
351: util::compressed_pair<constructor_type, allocator_type> pair_{};
353: using align_type = typename allocator_type::align_type;
354: using size_type = typename allocator_type::size_type;
356: friend base_type;
357: PetscErrorCode finalize_() noexcept;
358: };
360: // ==========================================================================================
361: // ObjectPool -- Private API
362: // ==========================================================================================
364: template <typename T, typename Constructor>
365: inline PetscErrorCode ObjectPool<T, Constructor>::finalize_() noexcept
366: {
367: PetscFunctionBegin;
368: {
369: auto _ = this->allocator().lock_guard();
371: // clang-format off
372: PetscCall(
373: this->allocator().for_each([&, this](void *ptr)
374: {
375: PetscFunctionBegin;
376: PetscCall(this->constructor().destroy(static_cast<value_type *>(ptr)));
377: PetscFunctionReturn(PETSC_SUCCESS);
378: })
379: );
380: // clang-format on
381: }
382: PetscCall(this->allocator().clear_());
383: PetscFunctionReturn(PETSC_SUCCESS);
384: }
386: // ==========================================================================================
387: // ObjectPool -- Public API
388: // ==========================================================================================
390: template <typename T, typename Constructor>
391: inline ObjectPool<T, Constructor>::~ObjectPool() noexcept
392: {
393: PetscFunctionBegin;
394: PetscCallAbort(PETSC_COMM_SELF, this->finalize());
395: PetscFunctionReturnVoid();
396: }
398: /*
399: ObjectPool::allocate - Allocate an object from the pool
401: Input Parameters:
402: . args... - The arguments to be passed to the constructor for the object
404: Output Parameter:
405: . obj - The pointer to the constructed object
407: Notes:
408: The user must deallocate the object using deallocate() and cannot free it themselves.
409: */
410: template <typename T, typename Constructor>
411: template <typename... Args>
412: inline PetscErrorCode ObjectPool<T, Constructor>::allocate(value_type **obj, Args &&...args) noexcept
413: {
414: auto allocated_from_pool = true;
415: void *mem = nullptr;
417: PetscFunctionBegin;
418: PetscAssertPointer(obj, 1);
419: // order is deliberate! We register our finalizer before the pool does so since we need to
420: // destroy the objects within it before it deletes their memory.
421: PetscCall(this->allocator().allocate(&mem, sizeof(value_type), static_cast<align_type>(alignof(value_type)), &allocated_from_pool));
422: PetscCall(this->register_finalize());
423: *obj = static_cast<value_type *>(mem);
424: if (allocated_from_pool) {
425: // if the allocation reused memory from the pool then this indicates the object is resettable.
426: PetscCall(this->constructor().reset(*obj, std::forward<Args>(args)...));
427: } else {
428: PetscCall(this->constructor().construct(*obj, std::forward<Args>(args)...));
429: }
430: PetscFunctionReturn(PETSC_SUCCESS);
431: }
433: /*
434: ObjectPool::deallocate - Return an object to the pool
436: Input Parameter:
437: . obj - The pointer to the object to return
439: Notes:
440: Sets obj to nullptr on return. obj must have been allocated by the pool in order to be
441: deallocated this way.
442: */
443: template <typename T, typename Constructor>
444: inline PetscErrorCode ObjectPool<T, Constructor>::deallocate(value_type **obj) noexcept
445: {
446: PetscFunctionBegin;
447: PetscAssertPointer(obj, 1);
448: PetscCall(this->register_finalize());
449: PetscCall(this->constructor().invalidate(*obj));
450: PetscCall(this->allocator().deallocate(reinterpret_cast<void **>(obj), sizeof(value_type), static_cast<align_type>(alignof(value_type))));
451: PetscFunctionReturn(PETSC_SUCCESS);
452: }
454: } // namespace Petsc