Halide  20.0.0
Halide compiler and libraries
vulkan_memory.h
Go to the documentation of this file.
1 #ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
2 #define HALIDE_RUNTIME_VULKAN_MEMORY_H
3 
6 #include "vulkan_internal.h"
7 
8 // Uncomment to enable verbose memory allocation debugging
9 // #define HL_VK_DEBUG_MEM 1
10 
11 namespace Halide {
12 namespace Runtime {
13 namespace Internal {
14 namespace Vulkan {
15 
16 // --------------------------------------------------------------------------
17 
18 // Enable external client to override Vulkan allocation callbacks (if they so desire)
20 WEAK const VkAllocationCallbacks *custom_allocation_callbacks = nullptr; // nullptr => use Vulkan runtime implementation
21 
22 // --------------------------------------------------------------------------
23 
24 // Runtime configuration parameters to adjust the behaviour of the block allocator
26  size_t maximum_pool_size = 0; //< Maximum number of bytes to allocate for the entire pool (including all blocks). Specified in bytes. Zero means no constraint
27  size_t minimum_block_size = 4 * 1024; //< Default block size is 4KB
28  size_t maximum_block_size = 0; //< Specified in bytes. Zero means no constraint
29  size_t maximum_block_count = 0; //< Maximum number of blocks to allocate. Zero means no constraint
30  size_t nearest_multiple = 32; //< Always round up the requested region sizes to the given integer value. Zero means no constraint
31 };
33 
34 // --------------------------------------------------------------------------
35 
36 /** Vulkan Memory Allocator class interface for managing large
37  * memory requests stored as contiguous blocks of memory, which
38  * are then sub-allocated into smaller regions of
39  * memory to avoid the excessive cost of vkAllocate and the limited
40  * number of available allocation calls through the API.
41  */
43 public:
44  // disable copy constructors and assignment
47 
48  // disable non-factory constrction
51 
52  // Factory methods for creation / destruction
53  static VulkanMemoryAllocator *create(void *user_context, const VulkanMemoryConfig &config,
54  VkDevice dev, VkPhysicalDevice phys_dev,
55  const SystemMemoryAllocatorFns &system_allocator,
56  const VkAllocationCallbacks *alloc_callbacks = nullptr);
57 
59 
60  // Public interface methods
61  MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
62  int conform(void *user_context, MemoryRequest *request); //< conforms the given memory request into one that can be allocated
63  int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
64  int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
65  int retain(void *user_context, MemoryRegion *region); //< retain the region and increase its use count
66  bool collect(void *user_context); //< returns true if any blocks were removed
67  int release(void *user_context);
68  int destroy(void *user_context);
69 
70  void *map(void *user_context, MemoryRegion *region);
71  int unmap(void *user_context, MemoryRegion *region);
73  int destroy_crop(void *user_context, MemoryRegion *region);
75 
76  VkDevice current_device() const {
77  return this->device;
78  }
79  VkPhysicalDevice current_physical_device() const {
80  return this->physical_device;
81  }
82  VkPhysicalDeviceLimits current_physical_device_limits() const {
83  return this->physical_device_limits;
84  }
85  const VkAllocationCallbacks *callbacks() const {
86  return this->alloc_callbacks;
87  }
88 
89  static const VulkanMemoryConfig &default_config();
90 
91  static int allocate_block(void *instance_ptr, MemoryBlock *block);
92  static int deallocate_block(void *instance_ptr, MemoryBlock *block);
93  static int conform_block_request(void *instance_ptr, MemoryRequest *request);
94 
95  static int allocate_region(void *instance_ptr, MemoryRegion *region);
96  static int deallocate_region(void *instance_ptr, MemoryRegion *region);
97  static int conform_region_request(void *instance_ptr, MemoryRequest *request);
98 
99  size_t bytes_allocated_for_blocks() const;
100  size_t blocks_allocated() const;
101 
102  size_t bytes_allocated_for_regions() const;
103  size_t regions_allocated() const;
104 
105 private:
106  static constexpr uint32_t invalid_memory_heap = uint32_t(-1);
107  static constexpr uint32_t invalid_usage_flags = uint32_t(-1);
108  static constexpr uint32_t invalid_memory_type = uint32_t(VK_MAX_MEMORY_TYPES);
109 
110  // Initializes a new instance
111  int initialize(void *user_context, const VulkanMemoryConfig &config,
112  VkDevice dev, VkPhysicalDevice phys_dev,
113  const SystemMemoryAllocatorFns &system_allocator,
114  const VkAllocationCallbacks *alloc_callbacks = nullptr);
115 
116  uint32_t select_memory_usage(void *user_context, MemoryProperties properties) const;
117 
118  uint32_t preferred_flags_for_memory_properties(void *user_context, MemoryProperties properties) const;
119  uint32_t required_flags_for_memory_properties(void *user_context, MemoryProperties properties) const;
120 
121  bool is_valid_memory_type_for_heap(void *user_context,
122  VkPhysicalDeviceMemoryProperties *memory_properties,
123  uint32_t memory_type_index,
124  uint32_t heap_index) const;
125 
126  bool is_preferred_memory_type_for_flags(void *user_context,
127  VkPhysicalDeviceMemoryProperties *memory_properties,
128  uint32_t memory_type_index,
129  uint32_t preferred_flags,
130  uint32_t required_flags) const;
131 
132  bool is_valid_memory_type_for_flags(void *user_context,
133  VkPhysicalDeviceMemoryProperties *memory_properties,
134  uint32_t memory_type_index,
135  uint32_t required_flags) const;
136 
137  int lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements);
138 
139  size_t block_byte_count = 0;
140  size_t block_count = 0;
141  size_t region_byte_count = 0;
142  size_t region_count = 0;
143  void *owner_context = nullptr;
144  VulkanMemoryConfig config;
145  VkDevice device = nullptr;
146  VkPhysicalDevice physical_device = nullptr;
147  VkPhysicalDeviceLimits physical_device_limits = {};
148  const VkAllocationCallbacks *alloc_callbacks = nullptr;
149  BlockAllocator *block_allocator = nullptr;
150 };
151 
153  const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
154  const SystemMemoryAllocatorFns &system_allocator,
155  const VkAllocationCallbacks *alloc_callbacks) {
156 
157  if (system_allocator.allocate == nullptr) {
158  error(user_context) << "VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
159  return nullptr;
160  }
161 
162  VulkanMemoryAllocator *result = reinterpret_cast<VulkanMemoryAllocator *>(
163  system_allocator.allocate(user_context, sizeof(VulkanMemoryAllocator)));
164 
165  if (result == nullptr) {
166  error(user_context) << "VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
167  return nullptr; // caller must handle error case for out-of-memory
168  }
169 
170  result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
171  return result;
172 }
173 
175  if (instance == nullptr) {
176  error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
178  }
179  BlockAllocator::MemoryAllocators allocators = instance->block_allocator->current_allocators();
180  instance->destroy(user_context);
181  if (allocators.system.deallocate == nullptr) {
182  error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
184  }
185  allocators.system.deallocate(user_context, instance);
187 }
188 
189 int VulkanMemoryAllocator::initialize(void *user_context,
190  const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
191  const SystemMemoryAllocatorFns &system_allocator,
192  const VkAllocationCallbacks *callbacks) {
193 
194  owner_context = user_context;
195  config = cfg;
196  device = dev;
197  physical_device = phys_dev;
198  alloc_callbacks = callbacks;
199  region_count = 0;
200  region_byte_count = 0;
201  block_count = 0;
202  block_byte_count = 0;
204  allocators.system = system_allocator;
207  BlockAllocator::Config block_allocator_config = {0};
208  block_allocator_config.maximum_pool_size = cfg.maximum_pool_size;
209  block_allocator_config.maximum_block_count = cfg.maximum_block_count;
210  block_allocator_config.maximum_block_size = cfg.maximum_block_size;
211  block_allocator_config.minimum_block_size = cfg.minimum_block_size;
212  block_allocator_config.nearest_multiple = cfg.nearest_multiple;
213  block_allocator = BlockAllocator::create(user_context, block_allocator_config, allocators);
214  if (block_allocator == nullptr) {
215  error(user_context) << "VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
217  }
218 
219  // get the physical device properties to determine limits and allocation requirements
220  VkPhysicalDeviceProperties physical_device_properties = {0};
221  memset(&physical_device_limits, 0, sizeof(VkPhysicalDeviceLimits));
222  vkGetPhysicalDeviceProperties(physical_device, &physical_device_properties);
223  memcpy(&physical_device_limits, &(physical_device_properties.limits), sizeof(VkPhysicalDeviceLimits));
225 }
226 
228 #if defined(HL_VK_DEBUG_MEM)
229  debug(nullptr) << "VulkanMemoryAllocator: Reserving memory ("
230  << "user_context=" << user_context << " "
231  << "block_allocator=" << (void *)(block_allocator) << " "
232  << "request_size=" << (uint32_t)(request.size) << " "
233  << "device=" << (void *)(device) << " "
234  << "physical_device=" << (void *)(physical_device) << ") ...\n";
235 #endif
236 
237  if ((device == nullptr) || (physical_device == nullptr)) {
238  error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
239  return nullptr;
240  }
241 
242  if (block_allocator == nullptr) {
243  error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
244  return nullptr;
245  }
246 
247  return block_allocator->reserve(this, request);
248 }
249 
251 #if defined(HL_VK_DEBUG_MEM)
252  debug(nullptr) << "VulkanMemoryAllocator: Mapping region ("
253  << "user_context=" << user_context << " "
254  << "device=" << (void *)(device) << " "
255  << "physical_device=" << (void *)(physical_device) << " "
256  << "region=" << (void *)(region) << " "
257  << "region_size=" << (uint32_t)region->size << " "
258  << "region_offset=" << (uint32_t)region->offset << " "
259  << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
260 #endif
261  if ((device == nullptr) || (physical_device == nullptr)) {
262  error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
263  return nullptr;
264  }
265 
266  if (block_allocator == nullptr) {
267  error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
268  return nullptr;
269  }
270 
271  MemoryRegion *owner = owner_of(user_context, region);
273  if (region_allocator == nullptr) {
274  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
275  return nullptr; // NOTE: caller must handle nullptr
276  }
277 
278  BlockResource *block_resource = region_allocator->block_resource();
279  if (block_resource == nullptr) {
280  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
281  return nullptr; // NOTE: caller must handle nullptr
282  }
283 
284  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
285  if (device_memory == nullptr) {
286  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
287  return nullptr; // NOTE: caller must handle nullptr
288  }
289 
290  void *mapped_ptr = nullptr;
291  VkDeviceSize memory_offset = region->offset + region->range.head_offset;
292  VkDeviceSize memory_size = region->size - region->range.tail_offset - region->range.head_offset;
293  if (((double)region->size - (double)region->range.tail_offset - (double)region->range.head_offset) <= 0.0) {
294  error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
295  return nullptr;
296  }
297 #if defined(HL_VK_DEBUG_MEM)
298  debug(nullptr) << "VulkanMemoryAllocator: MapMemory ("
299  << "user_context=" << user_context << "\n"
300  << " region_size=" << (uint32_t)region->size << "\n"
301  << " region_offset=" << (uint32_t)region->offset << "\n"
302  << " region_range.head_offset=" << (uint32_t)region->range.head_offset << "\n"
303  << " region_range.tail_offset=" << (uint32_t)region->range.tail_offset << "\n"
304  << " memory_offset=" << (uint32_t)memory_offset << "\n"
305  << " memory_size=" << (uint32_t)memory_size << "\n)\n";
306 #endif
307  VkResult result = vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (void **)(&mapped_ptr));
308  if (result != VK_SUCCESS) {
309  error(user_context) << "VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) << "\n";
310  return nullptr;
311  }
312 
313  return mapped_ptr;
314 }
315 
317 #if defined(HL_VK_DEBUG_MEM)
318  debug(nullptr) << "VulkanMemoryAllocator: Unmapping region ("
319  << "user_context=" << user_context << " "
320  << "device=" << (void *)(device) << " "
321  << "physical_device=" << (void *)(physical_device) << " "
322  << "region=" << (void *)(region) << " "
323  << "region_size=" << (uint32_t)region->size << " "
324  << "region_offset=" << (uint32_t)region->offset << " "
325  << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
326 #endif
327  if ((device == nullptr) || (physical_device == nullptr)) {
328  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
330  }
331 
332  MemoryRegion *owner = owner_of(user_context, region);
334  if (region_allocator == nullptr) {
335  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
337  }
338 
339  BlockResource *block_resource = region_allocator->block_resource();
340  if (block_resource == nullptr) {
341  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
343  }
344 
345  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
346  if (device_memory == nullptr) {
347  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
349  }
350 
351  vkUnmapMemory(device, *device_memory);
353 }
354 
356 #if defined(HL_VK_DEBUG_MEM)
357  debug(nullptr) << "VulkanMemoryAllocator: Cropping region ("
358  << "user_context=" << user_context << " "
359  << "device=" << (void *)(device) << " "
360  << "physical_device=" << (void *)(physical_device) << " "
361  << "region=" << (void *)(region) << " "
362  << "region_size=" << (uint32_t)region->size << " "
363  << "region_offset=" << (uint32_t)region->offset << " "
364  << "crop_offset=" << (int64_t)offset << ") ...\n";
365 #endif
366  if ((device == nullptr) || (physical_device == nullptr)) {
367  error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
368  return nullptr;
369  }
370 
371  MemoryRegion *owner = owner_of(user_context, region);
373  if (region_allocator == nullptr) {
374  error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
375  return nullptr; // NOTE: caller must handle nullptr
376  }
377 
378  // increment usage count
379  int error_code = region_allocator->retain(this, owner);
381  error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
382  return nullptr; // NOTE: caller must handle nullptr
383  }
384 
385  // create a new region to return, and copy all the other region's properties
386  const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
387  if (allocators.system.allocate == nullptr) {
388  error(user_context) << "VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
389  return nullptr;
390  }
391 
392  MemoryRegion *memory_region = reinterpret_cast<MemoryRegion *>(
393  allocators.system.allocate(user_context, sizeof(MemoryRegion)));
394 
395  if (memory_region == nullptr) {
396  error(user_context) << "VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
397  return nullptr;
398  }
399  memcpy(memory_region, owner, sizeof(MemoryRegion));
400 
401  // point the handle to the owner of the allocated region, and update the head offset
402  memory_region->is_owner = false;
403  memory_region->handle = (void *)owner;
404  memory_region->range.head_offset = owner->range.head_offset + offset;
405  return memory_region;
406 }
407 
409  if (region == nullptr) {
410  error(user_context) << "VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
412  }
413 
414  MemoryRegion *owner = owner_of(user_context, region);
416  if (region_allocator == nullptr) {
417  error(user_context) << "VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
419  }
420 
421  // decrement usage count
422  int error_code = region_allocator->release(this, owner);
424  error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
425  return error_code;
426  }
427 
428  // discard the copied region struct
429  const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
430  if (allocators.system.deallocate == nullptr) {
431  error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
433  }
434  allocators.system.deallocate(user_context, region);
436 }
437 
439  if (region->is_owner) {
440  return region;
441  } else {
442  // If this is a cropped region, use the handle to retrieve the owner of the allocation
443  return reinterpret_cast<MemoryRegion *>(region->handle);
444  }
445 }
446 
448 #if defined(HL_VK_DEBUG_MEM)
449  debug(nullptr) << "VulkanMemoryAllocator: Releasing region ("
450  << "user_context=" << user_context << " "
451  << "region=" << (void *)(region) << " "
452  << "size=" << (uint32_t)region->size << " "
453  << "offset=" << (uint32_t)region->offset << ") ...\n";
454 #endif
455  if ((device == nullptr) || (physical_device == nullptr)) {
456  error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
458  }
459  if (block_allocator == nullptr) {
460  error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
462  }
463  return block_allocator->release(this, region);
464 }
465 
467 #if defined(HL_VK_DEBUG_MEM)
468  debug(nullptr) << "VulkanMemoryAllocator: Reclaiming region ("
469  << "user_context=" << user_context << " "
470  << "region=" << (void *)(region) << " "
471  << "size=" << (uint32_t)region->size << " "
472  << "offset=" << (uint32_t)region->offset << ") ...\n";
473 #endif
474  if ((device == nullptr) || (physical_device == nullptr)) {
475  error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
477  }
478  if (block_allocator == nullptr) {
479  error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
481  }
482  return block_allocator->reclaim(this, region);
483 }
484 
486 #if defined(HL_VK_DEBUG_MEM)
487  debug(nullptr) << "VulkanMemoryAllocator: Retaining region ("
488  << "user_context=" << user_context << " "
489  << "region=" << (void *)(region) << " "
490  << "size=" << (uint32_t)region->size << " "
491  << "offset=" << (uint32_t)region->offset << ") ...\n";
492 #endif
493  if ((device == nullptr) || (physical_device == nullptr)) {
494  error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
496  }
497  if (block_allocator == nullptr) {
498  error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
500  }
501  return block_allocator->retain(this, region);
502 }
503 
505 #if defined(HL_VK_DEBUG_MEM)
506  debug(nullptr) << "VulkanMemoryAllocator: Collecting unused memory ("
507  << "user_context=" << user_context << ") ... \n";
508 #endif
509  if ((device == nullptr) || (physical_device == nullptr) || (block_allocator == nullptr)) {
510  return false;
511  }
512  return block_allocator->collect(this);
513 }
514 
516 #if defined(HL_VK_DEBUG_MEM)
517  debug(nullptr) << "VulkanMemoryAllocator: Releasing block allocator ("
518  << "user_context=" << user_context << ") ... \n";
519 #endif
520  if ((device == nullptr) || (physical_device == nullptr)) {
521  error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
523  }
524  if (block_allocator == nullptr) {
525  error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
527  }
528 
529  return block_allocator->release(this);
530 }
531 
533 #if defined(HL_VK_DEBUG_MEM)
534  debug(nullptr) << "VulkanMemoryAllocator: Destroying allocator ("
535  << "user_context=" << user_context << ") ... \n";
536 #endif
537  if (block_allocator != nullptr) {
538  BlockAllocator::destroy(user_context, block_allocator);
539  block_allocator = nullptr;
540  }
541  region_count = 0;
542  region_byte_count = 0;
543  block_count = 0;
544  block_byte_count = 0;
546 }
547 
548 const VulkanMemoryConfig &
550  static VulkanMemoryConfig result;
551  return result;
552 }
553 
554 // --
555 int VulkanMemoryAllocator::lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements) {
556 #if defined(HL_VK_DEBUG_MEM)
557  debug(nullptr) << "VulkanMemoryAllocator: Looking up requirements ("
558  << "user_context=" << user_context << " "
559  << "size=" << (uint32_t)block->size << ", "
560  << "usage_flags=" << usage_flags << ") ... \n";
561 #endif
562  VkBufferCreateInfo create_info = {
563  VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, // struct type
564  nullptr, // struct extending this
565  0, // create flags
566  size, // buffer size (in bytes)
567  usage_flags, // buffer usage flags
568  VK_SHARING_MODE_EXCLUSIVE, // sharing mode
569  0, nullptr};
570 
571  // Create a buffer to determine alignment requirements
572  VkBuffer buffer = VK_NULL_HANDLE;
573  VkResult result = vkCreateBuffer(this->device, &create_info, this->alloc_callbacks, &buffer);
574  if (result != VK_SUCCESS) {
575 #if defined(HL_VK_DEBUG_MEM)
576  debug(nullptr) << "VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
577  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
578 #endif
579  error(user_context) << "VulkanRegionAllocator: Failed to create buffer to gather memory requirements!\n";
581  }
582 
583  vkGetBufferMemoryRequirements(this->device, buffer, memory_requirements);
584  vkDestroyBuffer(this->device, buffer, this->alloc_callbacks);
586 }
587 
588 int VulkanMemoryAllocator::conform_block_request(void *instance_ptr, MemoryRequest *request) {
589 
590  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
591  if (instance == nullptr) {
593  }
594 
595  void *user_context = instance->owner_context;
596 #if defined(HL_VK_DEBUG_MEM)
597  debug(nullptr) << "VulkanMemoryAllocator: Conforming block request ("
598  << "user_context=" << user_context << " "
599  << "request=" << (void *)(request) << ") ... \n";
600 #endif
601 
602  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
603  error(user_context) << "VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
605  }
606 
607  VkMemoryRequirements memory_requirements = {0};
608  uint32_t usage_flags = instance->select_memory_usage(user_context, request->properties);
609  int error_code = instance->lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
611  error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
612  return error_code;
613  }
614 
615 #if defined(HL_VK_DEBUG_MEM)
616  debug(nullptr) << "VulkanMemoryAllocator: Block allocated ("
617  << "size=" << (uint32_t)request->size << ", "
618  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
619  << "required_size=" << (uint32_t)memory_requirements.size << ", "
620  << "uniform_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minUniformBufferOffsetAlignment << ", "
621  << "storage_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minStorageBufferOffsetAlignment << ", "
622  << "dedicated=" << (request->dedicated ? "true" : "false") << ")\n";
623 #endif
624 
625  request->size = memory_requirements.size;
626  request->properties.alignment = memory_requirements.alignment;
628 }
629 
630 int VulkanMemoryAllocator::allocate_block(void *instance_ptr, MemoryBlock *block) {
631  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
632  if (instance == nullptr) {
634  }
635 
636  void *user_context = instance->owner_context;
637  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
638  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
640  }
641 
642  if (block == nullptr) {
643  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
645  }
646 
647 #if defined(HL_VK_DEBUG_MEM)
648  debug(nullptr) << "VulkanMemoryAllocator: Allocating block ("
649  << "user_context=" << user_context << " "
650  << "block=" << (void *)(block) << " "
651  << "size=" << (uint64_t)block->size << ", "
652  << "dedicated=" << (block->dedicated ? "true" : "false") << " "
653  << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
654  << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
655  << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
656 #endif
657 
658  VkDeviceMemory *device_memory = (VkDeviceMemory *)vk_host_malloc(nullptr, sizeof(VkDeviceMemory), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
659  if (device_memory == nullptr) {
660  error(user_context) << "VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
662  }
663 
664  VkPhysicalDeviceMemoryProperties memory_properties;
665  vkGetPhysicalDeviceMemoryProperties(instance->physical_device, &memory_properties);
666 
667  // Attempt to allocate device memory with the requested flags, trying each type until successful
668  uint32_t selected_type = invalid_memory_type;
669  uint32_t preferred_flags = instance->preferred_flags_for_memory_properties(user_context, block->properties);
670  uint32_t required_flags = instance->required_flags_for_memory_properties(user_context, block->properties);
671 
672  // Try preferred flags first
673  for (uint32_t type_index = 0; type_index < memory_properties.memoryTypeCount; type_index++) {
674  if (!instance->is_preferred_memory_type_for_flags(user_context, &memory_properties, type_index, preferred_flags, required_flags)) {
675  continue;
676  }
677 
678  // Allocate memory
679  VkMemoryAllocateInfo alloc_info = {
680  VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // struct type
681  nullptr, // struct extending this
682  block->size, // size of allocation in bytes
683  type_index // memory type index from physical device
684  };
685 
686  VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
687  if (result == VK_SUCCESS) {
688  selected_type = type_index;
689  break;
690  }
691 
692 #if defined(HL_VK_DEBUG_MEM)
693  debug(user_context) << "VulkanMemoryAllocator: Allocation request for preferred flags was not successful (\n"
694  << "\tblock=" << (void *)(block) << "\n"
695  << "\ttype_index=" << type_index << "\n"
696  << "\tsize=" << (uint64_t)block->size << "\n"
697  << "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
698  << "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
699  << "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
700  << "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
701  << "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
702  << "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
703  << "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
704  << "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n"
705  << "vkAllocateMemory returned: "
706  << vk_get_error_name(result) << "\n";
707 #endif
708  }
709 
710  // try any valid type on any valid pool
711  if (selected_type == invalid_memory_type) {
712  for (uint32_t type_index = 0; type_index < memory_properties.memoryTypeCount; type_index++) {
713  if (!instance->is_valid_memory_type_for_flags(user_context, &memory_properties, type_index, required_flags)) {
714  continue;
715  }
716 
717  // Allocate memory
718  VkMemoryAllocateInfo alloc_info = {
719  VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // struct type
720  nullptr, // struct extending this
721  block->size, // size of allocation in bytes
722  type_index // memory type index from physical device
723  };
724 
725  VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
726  if (result == VK_SUCCESS) {
727  selected_type = type_index;
728  break;
729  }
730 #if defined(HL_VK_DEBUG_MEM)
731  debug(user_context) << "VulkanMemoryAllocator: Allocation request for valid flags was not successful (\n"
732  << "\tblock=" << (void *)(block) << "\n"
733  << "\ttype_index=" << type_index << "\n"
734  << "\tsize=" << (uint64_t)block->size << "\n"
735  << "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
736  << "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
737  << "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
738  << "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
739  << "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
740  << "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
741  << "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
742  << "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n"
743  << "vkAllocateMemory returned: "
744  << vk_get_error_name(result) << "\n";
745 #endif
746  }
747  }
748 
749  if (selected_type == invalid_memory_type) {
750  vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
751  device_memory = VK_NULL_HANDLE;
752  error(user_context) << "VulkanMemoryAllocator: Allocation failed for block (\n"
753  << "\tblock=" << (void *)(block) << "\n"
754  << "\tsize=" << (uint64_t)block->size << "\n"
755  << "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
756  << "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
757  << "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
758  << "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
759  << "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
760  << "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
761  << "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
762  << "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n";
764  }
765 
766 #ifdef DEBUG_RUNTIME
767  debug(nullptr) << "vkAllocateMemory: Allocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
768 #endif
769 
770  block->handle = (void *)device_memory;
771  instance->block_byte_count += block->size;
772  instance->block_count++;
774 }
775 
776 int VulkanMemoryAllocator::deallocate_block(void *instance_ptr, MemoryBlock *block) {
777  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
778  if (instance == nullptr) {
780  }
781 
782  void *user_context = instance->owner_context;
783 #if defined(HL_VK_DEBUG_MEM)
784  debug(nullptr) << "VulkanMemoryAllocator: Deallocating block ("
785  << "user_context=" << user_context << " "
786  << "block=" << (void *)(block) << ") ... \n";
787 #endif
788 
789  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
790  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
792  }
793 
794  if (block == nullptr) {
795  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
797  }
798 
799 #if defined(HL_VK_DEBUG_MEM)
800  debug(nullptr) << "VulkanBlockAllocator: deallocating block ("
801  << "size=" << (uint32_t)block->size << ", "
802  << "dedicated=" << (block->dedicated ? "true" : "false") << " "
803  << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
804  << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
805  << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
806 #endif
807 
808  // Nothing to do if device memory was already freed
809  if (block->handle == nullptr) {
811  }
812 
813  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block->handle);
814  if (device_memory == nullptr) {
815  error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
817  }
818 
819  vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
820 #ifdef DEBUG_RUNTIME
821  debug(nullptr) << "vkFreeMemory: Deallocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
822 #endif
823 
824  if (instance->block_count > 0) {
825  instance->block_count--;
826  } else {
827  error(nullptr) << "VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
828  instance->block_count = 0;
829  }
830 
831  if (int64_t(instance->block_byte_count) - int64_t(block->size) >= 0) {
832  instance->block_byte_count -= block->size;
833  } else {
834  error(nullptr) << "VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
835  instance->block_byte_count = 0;
836  }
837 
838  block->handle = nullptr;
839  vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
840  device_memory = nullptr;
842 }
843 
844 size_t VulkanMemoryAllocator::blocks_allocated() const {
845  return block_count;
846 }
847 
848 size_t VulkanMemoryAllocator::bytes_allocated_for_blocks() const {
849  return block_byte_count;
850 }
851 
852 uint32_t VulkanMemoryAllocator::required_flags_for_memory_properties(
853  void *user_context, MemoryProperties properties) const {
854 
855  uint32_t required_flags = 0; //< must have in order to enable requested access
856  switch (properties.visibility) {
858  required_flags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
859  break;
861  required_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
862  break;
864  required_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
865  break;
868  break;
870  error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
871  << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
872  return invalid_memory_type;
873  };
874  return required_flags;
875 }
876 
877 uint32_t VulkanMemoryAllocator::preferred_flags_for_memory_properties(
878  void *user_context, MemoryProperties properties) const {
879 
880  uint32_t preferred_flags = 0; //< preferred memory flags for requested access type
881  switch (properties.visibility) {
883  preferred_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
884  break;
886  preferred_flags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
887  break;
891  break;
893  error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
894  << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
895  return invalid_memory_type;
896  };
897 
898  uint32_t required_flags = required_flags_for_memory_properties(user_context, properties);
899  switch (properties.caching) {
901  if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
902  preferred_flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
903  }
904  break;
906  if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
907  preferred_flags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
908  }
909  break;
911  if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
912  preferred_flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
913  }
914  break;
917  break;
919  default:
920  error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
921  << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
922  return invalid_memory_type;
923  };
924  return preferred_flags;
925 }
926 
927 bool VulkanMemoryAllocator::is_valid_memory_type_for_heap(void *user_context,
928  VkPhysicalDeviceMemoryProperties *memory_properties,
929  uint32_t memory_type_index,
930  uint32_t heap_index) const {
931 
932  if (memory_type_index < memory_properties->memoryTypeCount) {
933  if (memory_properties->memoryTypes[memory_type_index].heapIndex == heap_index) {
934  return true;
935  }
936  }
937  return false;
938 }
939 
940 bool VulkanMemoryAllocator::is_valid_memory_type_for_flags(void *user_context,
941  VkPhysicalDeviceMemoryProperties *memory_properties,
942  uint32_t memory_type_index,
943  uint32_t required_flags) const {
944 
945  if (memory_type_index < memory_properties->memoryTypeCount) {
946  const VkMemoryPropertyFlags property_flags = memory_properties->memoryTypes[memory_type_index].propertyFlags;
947  if (required_flags) {
948  if ((property_flags & required_flags) != required_flags) {
949  return false;
950  }
951  }
952  return true;
953  }
954  return false;
955 }
956 
957 bool VulkanMemoryAllocator::is_preferred_memory_type_for_flags(void *user_context,
958  VkPhysicalDeviceMemoryProperties *memory_properties,
959  uint32_t memory_type_index,
960  uint32_t preferred_flags,
961  uint32_t required_flags) const {
962 
963  if (memory_type_index < memory_properties->memoryTypeCount) {
964  const VkMemoryPropertyFlags property_flags = memory_properties->memoryTypes[memory_type_index].propertyFlags;
965  if (required_flags) {
966  if ((property_flags & required_flags) != required_flags) {
967  return false;
968  }
969  }
970 
971  if (preferred_flags) {
972  if ((property_flags & preferred_flags) != preferred_flags) {
973  return false;
974  }
975  }
976  return true;
977  }
978  return false;
979 }
980 
981 // --
982 
983 int VulkanMemoryAllocator::conform(void *user_context, MemoryRequest *request) {
984 
985  // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
986  // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
987  // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
988  // whose size may be required to larger than our requested size (even though we will only ever touch the
989  // size of the region we're managing as within our block)
990 
991  VkMemoryRequirements memory_requirements = {0};
992  uint32_t usage_flags = select_memory_usage(user_context, request->properties);
993  int error_code = lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
995  error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
996  return error_code;
997  }
998 
999 #if defined(HL_VK_DEBUG_MEM)
1000  debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1001  << "requested_size=" << (uint32_t)region->size << ", "
1002  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1003  << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1004 #endif
1005 
1006  // Enforce any alignment constraints reported by the device limits for each usage type
1007  if (usage_flags & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) {
1008  if ((request->alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
1009  request->alignment = this->physical_device_limits.minStorageBufferOffsetAlignment;
1010  }
1011  } else if (usage_flags & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) {
1012  if ((request->alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
1013  request->alignment = this->physical_device_limits.minUniformBufferOffsetAlignment;
1014  }
1015  }
1016 
1017  // Ensure the request ends on an aligned address
1018  if (request->alignment > config.nearest_multiple) {
1019  request->properties.nearest_multiple = request->alignment;
1020  }
1021 
1022  size_t actual_alignment = conform_alignment(request->alignment, memory_requirements.alignment);
1023  size_t actual_offset = aligned_offset(request->offset, actual_alignment);
1024  size_t actual_size = conform_size(actual_offset, memory_requirements.size, actual_alignment, request->properties.nearest_multiple);
1025 
1026 #if defined(HL_VK_DEBUG_MEM)
1027  if ((request->size != actual_size) || (request->alignment != actual_alignment) || (request->offset != actual_offset)) {
1028  debug(nullptr) << "VulkanMemoryAllocator: Adjusting request to match requirements (\n"
1029  << " size = " << (uint64_t)request->size << " => " << (uint64_t)actual_size << ",\n"
1030  << " alignment = " << (uint64_t)request->alignment << " => " << (uint64_t)actual_alignment << ",\n"
1031  << " offset = " << (uint64_t)request->offset << " => " << (uint64_t)actual_offset << ",\n"
1032  << " required.size = " << (uint64_t)memory_requirements.size << ",\n"
1033  << " required.alignment = " << (uint64_t)memory_requirements.alignment << "\n)\n";
1034  }
1035 #endif
1036  request->size = actual_size;
1037  request->alignment = actual_alignment;
1038  request->offset = actual_offset;
1039 
1041 }
1042 
1043 int VulkanMemoryAllocator::conform_region_request(void *instance_ptr, MemoryRequest *request) {
1044 
1045  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1046  if (instance == nullptr) {
1048  }
1049 
1050  void *user_context = instance->owner_context;
1051 #if defined(HL_VK_DEBUG_MEM)
1052  debug(nullptr) << "VulkanMemoryAllocator: Conforming region request ("
1053  << "user_context=" << user_context << " "
1054  << "request=" << (void *)(region) << ") ... \n";
1055 #endif
1056 
1057  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1058  error(user_context) << "VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
1060  }
1061 
1062 #if defined(HL_VK_DEBUG_MEM)
1063  debug(nullptr) << "VulkanRegionAllocator: Conforming region request ("
1064  << "size=" << (uint32_t)request->size << ", "
1065  << "offset=" << (uint32_t)request->offset << ", "
1066  << "dedicated=" << (request->dedicated ? "true" : "false") << " "
1067  << "usage=" << halide_memory_usage_name(request->properties.usage) << " "
1068  << "caching=" << halide_memory_caching_name(request->properties.caching) << " "
1069  << "visibility=" << halide_memory_visibility_name(request->properties.visibility) << ")\n";
1070 #endif
1071 
1072  return instance->conform(user_context, request);
1073 }
1074 
1075 int VulkanMemoryAllocator::allocate_region(void *instance_ptr, MemoryRegion *region) {
1076 
1077  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1078  if (instance == nullptr) {
1080  }
1081 
1082  void *user_context = instance->owner_context;
1083 #if defined(HL_VK_DEBUG_MEM)
1084  debug(nullptr) << "VulkanMemoryAllocator: Allocating region ("
1085  << "user_context=" << user_context << " "
1086  << "region=" << (void *)(region) << ") ... \n";
1087 #endif
1088 
1089  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1090  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
1092  }
1093 
1094  if (region == nullptr) {
1095  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
1097  }
1098 
1099 #if defined(HL_VK_DEBUG_MEM)
1100  debug(nullptr) << "VulkanRegionAllocator: Allocating region ("
1101  << "size=" << (uint32_t)region->size << ", "
1102  << "offset=" << (uint32_t)region->offset << ", "
1103  << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1104  << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1105  << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1106  << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1107 #endif
1108 
1109  uint32_t usage_flags = instance->select_memory_usage(user_context, region->properties);
1110 
1111  VkBufferCreateInfo create_info = {
1112  VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, // struct type
1113  nullptr, // struct extending this
1114  0, // create flags
1115  region->size, // buffer size (in bytes)
1116  usage_flags, // buffer usage flags
1117  VK_SHARING_MODE_EXCLUSIVE, // sharing mode
1118  0, nullptr};
1119 
1120  VkBuffer *buffer = (VkBuffer *)vk_host_malloc(nullptr, sizeof(VkBuffer), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
1121  if (buffer == nullptr) {
1122  error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
1124  }
1125 
1126  VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1127  if (result != VK_SUCCESS) {
1128  // Allocation failed ... collect unused regions and try again ...
1129  instance->collect(user_context);
1130  result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1131  }
1132  if (result != VK_SUCCESS) {
1133  vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1134  buffer = nullptr;
1135  error(user_context) << "VulkanRegionAllocator: Failed to create buffer!\n\t"
1136  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1138  }
1139 
1140  // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
1141  // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
1142  // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
1143  // whose size may be required to larger than our requested size (even though we will only ever touch the
1144  // size of the region we're managing as within our block)
1145  VkMemoryRequirements memory_requirements = {0};
1146  vkGetBufferMemoryRequirements(instance->device, *buffer, &memory_requirements);
1147 
1148 #if defined(HL_VK_DEBUG_MEM)
1149  debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1150  << "requested_size=" << (uint32_t)region->size << ", "
1151  << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1152  << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1153 #endif
1154 
1155  if (memory_requirements.size > region->size) {
1156  vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1157 #ifdef DEBUG_RUNTIME
1158  debug(nullptr) << "VulkanMemoryAllocator: Reallocating buffer to match required size ("
1159  << (uint64_t)region->size << " => " << (uint64_t)memory_requirements.size << " bytes) ...\n";
1160 #endif
1161  create_info.size = memory_requirements.size;
1162  VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1163  if (result != VK_SUCCESS) {
1164  error(user_context) << "VulkanRegionAllocator: Failed to recreate buffer!\n\t"
1165  << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1167  }
1168  }
1169 
1170 #ifdef DEBUG_RUNTIME
1171  debug(nullptr) << "vkCreateBuffer: Created buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1172 #endif
1173 
1174  RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, region);
1175  if (region_allocator == nullptr) {
1176  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
1178  }
1179 
1180  BlockResource *block_resource = region_allocator->block_resource();
1181  if (block_resource == nullptr) {
1182  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
1184  }
1185 
1186  VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
1187  if (device_memory == nullptr) {
1188  error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
1190  }
1191 
1192  // Finally, bind buffer to the device memory
1193  result = vkBindBufferMemory(instance->device, *buffer, *device_memory, region->offset);
1194  if (result != VK_SUCCESS) {
1195  error(user_context) << "VulkanRegionAllocator: Failed to bind buffer!\n\t"
1196  << "vkBindBufferMemory returned: " << vk_get_error_name(result) << "\n";
1198  }
1199 
1200  region->handle = (void *)buffer;
1201  region->is_owner = true;
1202  instance->region_byte_count += region->size;
1203  instance->region_count++;
1205 }
1206 
1207 int VulkanMemoryAllocator::deallocate_region(void *instance_ptr, MemoryRegion *region) {
1208  VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1209  if (instance == nullptr) {
1211  }
1212 
1213  void *user_context = instance->owner_context;
1214 #if defined(HL_VK_DEBUG_MEM)
1215  debug(nullptr) << "VulkanMemoryAllocator: Deallocating region ("
1216  << "user_context=" << user_context << " "
1217  << "region=" << (void *)(region) << ") ... \n";
1218 #endif
1219 
1220  if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1221  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
1223  }
1224 
1225  if (region == nullptr) {
1226  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
1228  }
1229 
1230 #if defined(HL_VK_DEBUG_MEM)
1231  debug(nullptr) << "VulkanRegionAllocator: Deallocating region ("
1232  << "size=" << (uint32_t)region->size << ", "
1233  << "offset=" << (uint32_t)region->offset << ", "
1234  << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1235  << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1236  << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1237  << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1238 #endif
1239 
1240  if (region->handle == nullptr) {
1241  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
1243  }
1244 
1245  VkBuffer *buffer = reinterpret_cast<VkBuffer *>(region->handle);
1246  if (buffer == nullptr) {
1247  error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
1249  }
1250 
1251  vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1252 #ifdef DEBUG_RUNTIME
1253  debug(nullptr) << "vkDestroyBuffer: Destroyed buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1254 #endif
1256  region->handle = nullptr;
1257  if (instance->region_count > 0) {
1258  instance->region_count--;
1259  } else {
1260  error(user_context) << "VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
1261  instance->region_count = 0;
1263  }
1264 
1265  if (int64_t(instance->region_byte_count) - int64_t(region->size) >= 0) {
1266  instance->region_byte_count -= region->size;
1267  } else {
1268  error(user_context) << "VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
1269  instance->region_byte_count = 0;
1271  }
1272  vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1273  buffer = nullptr;
1274  return error_code;
1275 }
1276 
1277 size_t VulkanMemoryAllocator::regions_allocated() const {
1278  return region_count;
1279 }
1280 
1281 size_t VulkanMemoryAllocator::bytes_allocated_for_regions() const {
1282  return region_byte_count;
1283 }
1284 
1285 uint32_t VulkanMemoryAllocator::select_memory_usage(void *user_context, MemoryProperties properties) const {
1286  uint32_t result = 0;
1287  switch (properties.usage) {
1289  result |= VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT;
1290  break;
1293  result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
1294  break;
1296  result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
1297  break;
1299  result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
1300  break;
1302  result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
1303  break;
1306  default:
1307  error(user_context) << "VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
1308  << "usage=" << halide_memory_usage_name(properties.usage) << "\n";
1309  return invalid_usage_flags;
1310  };
1311 
1312  if (result == invalid_usage_flags) {
1313  error(user_context) << "VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
1314  << "usage=" << halide_memory_usage_name(properties.usage) << " "
1315  << "caching=" << halide_memory_caching_name(properties.caching) << " "
1316  << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
1317  return invalid_usage_flags;
1318  }
1319 
1320  return result;
1321 }
1322 
1323 // --------------------------------------------------------------------------
1324 
1325 namespace {
1326 
1327 // --------------------------------------------------------------------------
1328 // Halide System allocator for host allocations
1329 void *vk_system_malloc(void *user_context, size_t size) {
1330  return malloc(size);
1331 }
1332 
1333 void vk_system_free(void *user_context, void *ptr) {
1334  free(ptr);
1335 }
1336 
1337 // Vulkan host-side allocation
1338 void *vk_host_malloc(void *user_context, size_t size, size_t alignment, VkSystemAllocationScope scope, const VkAllocationCallbacks *callbacks) {
1339  if (callbacks) {
1340  return callbacks->pfnAllocation(user_context, size, alignment, scope);
1341  } else {
1342  return vk_system_malloc(user_context, size);
1343  }
1344 }
1345 
1346 void vk_host_free(void *user_context, void *ptr, const VkAllocationCallbacks *callbacks) {
1347  if (callbacks) {
1348  return callbacks->pfnFree(user_context, ptr);
1349  } else {
1350  return vk_system_free(user_context, ptr);
1351  }
1352 }
1353 
1354 VulkanMemoryAllocator *vk_create_memory_allocator(void *user_context,
1355  VkDevice device,
1356  VkPhysicalDevice physical_device,
1357  const VkAllocationCallbacks *alloc_callbacks) {
1358 
1359  SystemMemoryAllocatorFns system_allocator = {vk_system_malloc, vk_system_free};
1360  VulkanMemoryConfig config = memory_allocator_config;
1361 
1362  // Parse the allocation config string (if specified).
1363  //
1364  // `HL_VK_ALLOC_CONFIG=N:N:N` will tell Halide to configure the Vulkan memory
1365  // allocator use the given constraints specified as three integer values
1366  // separated by a `:` or `;`. These values correspond to `minimum_block_size`,
1367  // `maximum_block_size` and `maximum_block_count`.
1368  //
1369  const char *alloc_config = vk_get_alloc_config_internal(user_context);
1371  StringTable alloc_config_values;
1372  alloc_config_values.parse(user_context, alloc_config, HL_VK_ENV_DELIM);
1373  if (alloc_config_values.size() > 0) {
1374  config.maximum_pool_size = atoi(alloc_config_values[0]) * 1024 * 1024;
1375  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_pool_size << " for maximum pool size (in bytes)\n";
1376  }
1377  if (alloc_config_values.size() > 1) {
1378  config.minimum_block_size = atoi(alloc_config_values[1]) * 1024 * 1024;
1379  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.minimum_block_size << " for minimum block size (in bytes)\n";
1380  }
1381  if (alloc_config_values.size() > 2) {
1382  config.maximum_block_size = atoi(alloc_config_values[2]) * 1024 * 1024;
1383  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_size << " for maximum block size (in bytes)\n";
1384  }
1385  if (alloc_config_values.size() > 3) {
1386  config.maximum_block_count = atoi(alloc_config_values[3]);
1387  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_count << " for maximum block count\n";
1388  }
1389  if (alloc_config_values.size() > 4) {
1390  config.nearest_multiple = atoi(alloc_config_values[4]);
1391  print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.nearest_multiple << " for nearest multiple\n";
1392  }
1393  }
1394 
1395  return VulkanMemoryAllocator::create(user_context,
1396  config, device, physical_device,
1397  system_allocator, alloc_callbacks);
1398 }
1399 
1400 int vk_destroy_memory_allocator(void *user_context, VulkanMemoryAllocator *allocator) {
1401  if (allocator != nullptr) {
1402  allocator->collect(user_context);
1404  allocator = nullptr;
1405  }
1407 }
1408 
1409 // --------------------------------------------------------------------------
1410 
1411 int vk_clear_device_buffer(void *user_context,
1412  VulkanMemoryAllocator *allocator,
1413  VkCommandBuffer command_buffer,
1414  VkQueue command_queue,
1415  VkBuffer device_buffer) {
1416 
1417 #ifdef DEBUG_RUNTIME
1418  debug(user_context)
1419  << " vk_clear_device_buffer (user_context: " << user_context << ", "
1420  << "allocator: " << (void *)allocator << ", "
1421  << "command_buffer: " << (void *)command_buffer << ", "
1422  << "command_queue: " << (void *)command_queue << ", "
1423  << "device_buffer: " << (void *)device_buffer << ")\n";
1424 #endif
1425 
1426  // begin the command buffer
1427  VkCommandBufferBeginInfo command_buffer_begin_info =
1428  {
1429  VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, // struct type
1430  nullptr, // pointer to struct extending this
1431  VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, // flags
1432  nullptr // pointer to parent command buffer
1433  };
1434 
1435  VkResult result = vkBeginCommandBuffer(command_buffer, &command_buffer_begin_info);
1436  if (result != VK_SUCCESS) {
1437  error(user_context) << "Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
1439  }
1440 
1441  // fill buffer with zero values up to the size of the buffer
1442  vkCmdFillBuffer(command_buffer, device_buffer, 0, VK_WHOLE_SIZE, 0);
1443 
1444  // end the command buffer
1445  result = vkEndCommandBuffer(command_buffer);
1446  if (result != VK_SUCCESS) {
1447  error(user_context) << "Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
1449  }
1450 
1451  // submit the command buffer
1452  VkSubmitInfo submit_info =
1453  {
1454  VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
1455  nullptr, // pointer to struct extending this
1456  0, // wait semaphore count
1457  nullptr, // semaphores
1458  nullptr, // pipeline stages where semaphore waits occur
1459  1, // how many command buffers to execute
1460  &command_buffer, // the command buffers
1461  0, // number of semaphores to signal
1462  nullptr // the semaphores to signal
1463  };
1464 
1465  result = vkQueueSubmit(command_queue, 1, &submit_info, VK_NULL_HANDLE);
1466  if (result != VK_SUCCESS) {
1467  error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
1469  }
1470 
1471  // wait for memset to finish
1472  result = vkQueueWaitIdle(command_queue);
1473  if (result != VK_SUCCESS) {
1474  error(user_context) << "Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) << "\n";
1476  }
1477 
1479 }
1480 
1481 // --------------------------------------------------------------------------
1482 
1483 } // namespace
1484 } // namespace Vulkan
1485 } // namespace Internal
1486 } // namespace Runtime
1487 } // namespace Halide
1488 
1489 // --------------------------------------------------------------------------
1490 
1491 extern "C" {
1492 
1493 // --------------------------------------------------------------------------
1494 
1495 WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks) {
1496  using namespace Halide::Runtime::Internal::Vulkan;
1498  custom_allocation_callbacks = callbacks;
1499 }
1500 
1501 WEAK const VkAllocationCallbacks *halide_vulkan_get_allocation_callbacks(void *user_context) {
1502  using namespace Halide::Runtime::Internal::Vulkan;
1505 }
1506 
1507 // --------------------------------------------------------------------------
1508 
1509 } // extern "C"
1510 
1511 #endif // HALIDE_RUNTIME_VULKAN_MEMORY_H
halide_error_code_t
The error codes that may be returned by a Halide pipeline.
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
@ halide_error_code_generic_error
An uncategorized error occurred.
@ halide_error_code_success
There was no error.
@ halide_error_code_device_malloc_failed
The Halide runtime encountered an error while trying to allocate memory on device.
@ halide_error_code_out_of_memory
A call to halide_malloc returned NULL.
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int release(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *region)
static void destroy(void *user_context, BlockAllocator *block_allocator)
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
int reclaim(void *user_context, MemoryRegion *region)
const MemoryAllocators & current_allocators() const
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
int retain(void *user_context, MemoryRegion *memory_region)
int release(void *user_context, MemoryRegion *memory_region)
size_t parse(void *user_context, const char *str, const char *delim)
Definition: string_table.h:159
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
Definition: vulkan_memory.h:42
int reclaim(void *user_context, MemoryRegion *region)
static const VulkanMemoryConfig & default_config()
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
VulkanMemoryAllocator(const VulkanMemoryAllocator &)=delete
int release(void *user_context, MemoryRegion *region)
static int allocate_block(void *instance_ptr, MemoryBlock *block)
int conform(void *user_context, MemoryRequest *request)
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
int unmap(void *user_context, MemoryRegion *region)
const VkAllocationCallbacks * callbacks() const
Definition: vulkan_memory.h:85
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
int destroy_crop(void *user_context, MemoryRegion *region)
static int allocate_region(void *instance_ptr, MemoryRegion *region)
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
VkPhysicalDeviceLimits current_physical_device_limits() const
Definition: vulkan_memory.h:82
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
void * map(void *user_context, MemoryRegion *region)
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
int retain(void *user_context, MemoryRegion *region)
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
WEAK const char * halide_memory_caching_name(MemoryCaching value)
void destroy(const T *t)
WEAK VulkanMemoryConfig memory_allocator_config
Definition: vulkan_memory.h:32
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
Definition: vulkan_memory.h:20
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
Definition: vulkan_memory.h:19
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
unsigned __INT64_TYPE__ uint64_t
signed __INT64_TYPE__ int64_t
void * malloc(size_t)
int atoi(const char *)
void * memset(void *s, int val, size_t n)
unsigned __INT32_TYPE__ uint32_t
#define WEAK
void * memcpy(void *s1, const void *s2, size_t n)
void free(void *)
static bool is_empty(const char *str)
#define HL_VK_ENV_DELIM
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)
int error_code
void * user_context
VkCommandBuffer command_buffer
VulkanMemoryAllocator * allocator