/*
 * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
 * SPDX-License-Identifier: MIT
 */
#include "nvk_image.h"

#include "nvk_device.h"
#include "nvk_device_memory.h"
#include "nvk_entrypoints.h"
#include "nvk_format.h"
#include "nvk_physical_device.h"
#include "nvkmd/nvkmd.h"

#include "util/detect_os.h"
#include "vk_android.h"
#include "vk_enum_to_str.h"
#include "vk_format.h"
#include "nil.h"
#include "vk_enum_defines.h"

#include "clb097.h"
#include "clb197.h"
#include "clc197.h"
#include "clc597.h"
#include "clcd97.h"

static bool
nvk_use_separate_zs(const struct nvk_physical_device *pdev, VkFormat vk_format)
{
   /* Separate depth/stencil doesn't exist pre-Blackwell */
   if (pdev->info.cls_eng3d < BLACKWELL_A)
      return false;

   const VkImageAspectFlags format_aspects = vk_format_aspects(vk_format);

   /* Just depth or just stencil is still a single plane */
   return format_aspects == (VK_IMAGE_ASPECT_DEPTH_BIT |
                             VK_IMAGE_ASPECT_STENCIL_BIT);
}

static VkFormatFeatureFlags2
nvk_get_image_plane_format_features(const struct nvk_physical_device *pdev,
                                    VkFormat vk_format, VkImageTiling tiling,
                                    uint64_t drm_format_mod)
{
   VkFormatFeatureFlags2 features = 0;

   if (tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT &&
       drm_format_mod != DRM_FORMAT_MOD_LINEAR &&
       !fourcc_mod_is_vendor(drm_format_mod, NVIDIA))
      return 0;

   enum pipe_format p_format = nvk_format_to_pipe_format(vk_format);
   if (p_format == PIPE_FORMAT_NONE)
      return 0;

   /* You can't tile a non-power-of-two */
   if (!util_is_power_of_two_nonzero(util_format_get_blocksize(p_format)))
      return 0;

   if (nil_format_supports_texturing(&pdev->info, p_format)) {
      features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
      features |= VK_FORMAT_FEATURE_2_BLIT_SRC_BIT;
   }

   if (nil_format_supports_filtering(&pdev->info, p_format)) {
      features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT;
      if (pdev->info.cls_eng3d >= MAXWELL_B)
         features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_MINMAX_BIT;
   }

   /* TODO: VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT */
   if (vk_format_has_depth(vk_format)) {
      features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT;
   }

   if (nil_format_supports_color_targets(&pdev->info, p_format) && 
       tiling != VK_IMAGE_TILING_LINEAR) {
      features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
      if (nil_format_supports_blending(&pdev->info, p_format))
         features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT;
      features |= VK_FORMAT_FEATURE_2_BLIT_DST_BIT;
   }

   if (vk_format_is_depth_or_stencil(vk_format)) {
      if (!nil_format_supports_depth_stencil(&pdev->info, p_format) ||
          tiling == VK_IMAGE_TILING_LINEAR)
         return 0;

      features |= VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
   }

   if (nvk_format_supports_storage(pdev, p_format)) {
      features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT |
                  VK_FORMAT_FEATURE_2_STORAGE_WRITE_WITHOUT_FORMAT_BIT;
      if (pdev->info.cls_eng3d >= MAXWELL_A)
         features |= VK_FORMAT_FEATURE_2_STORAGE_READ_WITHOUT_FORMAT_BIT;
   }

   if (nvk_format_supports_atomics(pdev, p_format))
      features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_ATOMIC_BIT;

   if (p_format == PIPE_FORMAT_R8_UINT && tiling == VK_IMAGE_TILING_OPTIMAL)
      features |= VK_FORMAT_FEATURE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR;

   if (features != 0) {
      features |= VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT;
      features |= VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
      if (!vk_format_is_depth_or_stencil(vk_format))
         features |= VK_FORMAT_FEATURE_2_HOST_IMAGE_TRANSFER_BIT_EXT;

      /* The copy engine handles depth and stencil just fine */
      if (vk_format_has_depth(vk_format)) {
         features |= VK_FORMAT_FEATURE_2_DEPTH_COPY_ON_COMPUTE_QUEUE_BIT_KHR |
                     VK_FORMAT_FEATURE_2_DEPTH_COPY_ON_TRANSFER_QUEUE_BIT_KHR;
      }
      if (vk_format_has_stencil(vk_format)) {
         features |= VK_FORMAT_FEATURE_2_STENCIL_COPY_ON_COMPUTE_QUEUE_BIT_KHR |
                     VK_FORMAT_FEATURE_2_STENCIL_COPY_ON_TRANSFER_QUEUE_BIT_KHR;
      }
   }

   return features;
}

VkFormatFeatureFlags2
nvk_get_image_format_features(const struct nvk_physical_device *pdev,
                              VkFormat vk_format, VkImageTiling tiling,
                              uint64_t drm_format_mod)
{
   const struct vk_format_ycbcr_info *ycbcr_info =
         vk_format_get_ycbcr_info(vk_format);
   if (ycbcr_info == NULL) {
      return nvk_get_image_plane_format_features(pdev, vk_format, tiling,
                                                 drm_format_mod);
   }

   /* For multi-plane, we get the feature flags of each plane separately,
    * then take their intersection as the overall format feature flags
    */
   VkFormatFeatureFlags2 features = ~0ull;
   bool cosited_chroma = false;
   for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
      const struct vk_format_ycbcr_plane *plane_info = &ycbcr_info->planes[plane];
      features &= nvk_get_image_plane_format_features(pdev, plane_info->format,
                                                      tiling, drm_format_mod);
      if (plane_info->denominator_scales[0] > 1 ||
          plane_info->denominator_scales[1] > 1)
         cosited_chroma = true;
   }
   if (features == 0)
      return 0;

   /* Uh... We really should be able to sample from YCbCr */
   assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT);
   assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT);

   /* These aren't allowed for YCbCr formats */
   features &= ~(VK_FORMAT_FEATURE_2_BLIT_SRC_BIT |
                 VK_FORMAT_FEATURE_2_BLIT_DST_BIT |
                 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
                 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT |
                 VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);

   /* This is supported on all YCbCr formats */
   features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT;

   if (ycbcr_info->n_planes > 1) {
      /* DISJOINT_BIT implies that each plane has its own separate binding,
       * while SEPARATE_RECONSTRUCTION_FILTER_BIT implies that luma and chroma
       * each have their own, separate filters, so these two bits make sense
       * for multi-planar formats only.
       *
       * For MIDPOINT_CHROMA_SAMPLES_BIT, NVIDIA HW on single-plane interleaved
       * YCbCr defaults to COSITED_EVEN, which is inaccurate and fails tests.
       * This can be fixed with a NIR tweak but for now, we only enable this bit
       * for multi-plane formats. See Issue #9525 on the mesa/main tracker.
       */
      features |= VK_FORMAT_FEATURE_DISJOINT_BIT |
                  VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_SEPARATE_RECONSTRUCTION_FILTER_BIT |
                  VK_FORMAT_FEATURE_2_MIDPOINT_CHROMA_SAMPLES_BIT;
   }

   if (cosited_chroma)
      features |= VK_FORMAT_FEATURE_COSITED_CHROMA_SAMPLES_BIT;

   return features;
}

void
nvk_get_drm_format_modifier_properties_list(const struct nvk_physical_device *pdev,
                                            VkFormat vk_format,
                                            VkBaseOutStructure *ext)
{
   assert(ext->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT ||
          ext->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT);

   /* The two top-level data structures are the same.  It's only when
    * you get to walking the actual list of modifier properties that
    * they differ.
    */
   VkDrmFormatModifierPropertiesListEXT *p = (void *)ext;

   /* We don't support modifiers for YCbCr images */
   if (vk_format_get_ycbcr_info(vk_format) != NULL) {
      p->drmFormatModifierCount = 0;
      return;
   }

   /* Check that we actually support the format so we don't try to query
    * modifiers for formats NIL doesn't support.
    */
   const VkFormatFeatureFlags2 tiled_features =
      nvk_get_image_plane_format_features(pdev, vk_format,
                                          VK_IMAGE_TILING_OPTIMAL,
                                          DRM_FORMAT_MOD_INVALID);
   if (tiled_features == 0) {
      p->drmFormatModifierCount = 0;
      return;
   }

   uint64_t mods[NIL_MAX_DRM_FORMAT_MODS];
   size_t mod_count = NIL_MAX_DRM_FORMAT_MODS;
   enum pipe_format p_format = nvk_format_to_pipe_format(vk_format);
   nil_drm_format_mods_for_format(&pdev->info, nil_format(p_format),
                                  &mod_count, &mods);
   if (mod_count == 0) {
      p->drmFormatModifierCount = 0;
      return;
   }

   switch (ext->sType) {
   case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT: {
      VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierPropertiesEXT, out,
                             p->pDrmFormatModifierProperties,
                             &p->drmFormatModifierCount);

      for (uint32_t i = 0; i < mod_count; i++) {
         const VkFormatFeatureFlags2 features2 =
            nvk_get_image_format_features(pdev, vk_format,
                                          VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
                                          mods[i]);
         if (features2 != 0) {
            vk_outarray_append_typed(VkDrmFormatModifierPropertiesEXT, &out, mp) {
               mp->drmFormatModifier = mods[i];
               mp->drmFormatModifierPlaneCount = 1;
               mp->drmFormatModifierTilingFeatures =
                  vk_format_features2_to_features(features2);
            }
         }
      }
      break;
   }

   case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT: {
      VkDrmFormatModifierPropertiesList2EXT *p2 = (void *)p;
      VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierProperties2EXT, out,
                             p2->pDrmFormatModifierProperties,
                             &p2->drmFormatModifierCount);

      for (uint32_t i = 0; i < mod_count; i++) {
         const VkFormatFeatureFlags2 features2 =
            nvk_get_image_format_features(pdev, vk_format,
                                          VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
                                          mods[i]);
         if (features2 != 0) {
            vk_outarray_append_typed(VkDrmFormatModifierProperties2EXT, &out, mp) {
               mp->drmFormatModifier = mods[i];
               mp->drmFormatModifierPlaneCount = 1;
               mp->drmFormatModifierTilingFeatures = features2;
            }
         }
      }
      break;
   }

   default:
      UNREACHABLE("Invalid structure type");
   }
}

static VkFormatFeatureFlags2
vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)
{
   assert(util_bitcount(usage_flag) == 1);
   switch (usage_flag) {
   case VK_IMAGE_USAGE_TRANSFER_SRC_BIT:
      return VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
             VK_FORMAT_FEATURE_BLIT_SRC_BIT;
   case VK_IMAGE_USAGE_TRANSFER_DST_BIT:
      return VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT |
             VK_FORMAT_FEATURE_BLIT_DST_BIT;
   case VK_IMAGE_USAGE_SAMPLED_BIT:
      return VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
   case VK_IMAGE_USAGE_STORAGE_BIT:
      return VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT;
   case VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT:
      return VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
   case VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT:
      return VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
   case VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT:
      return VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
             VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
   case VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR:
      return VK_FORMAT_FEATURE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR;
   default:
      return 0;
   }
}

uint32_t
nvk_image_max_dimension(const struct nv_device_info *info,
                        VkImageType image_type)
{
   switch (image_type) {
   case VK_IMAGE_TYPE_1D:
   case VK_IMAGE_TYPE_2D:
      /* The render and texture units can support up to 16K all the way back
       * to Kepler but the copy engine can't.  We can work around this by
       * doing offset shenanigans in the copy code but that not currently
       * implemented.
       */
      return info->cls_eng3d >= PASCAL_B ? 0x8000 : 0x4000;
   case VK_IMAGE_TYPE_3D:
      return 0x4000;
   default:
      UNREACHABLE("Invalid image type");
   }
}

static uint64_t
get_explicit_drm_format_mod(const void *pNext)
{
   const VkPhysicalDeviceImageDrmFormatModifierInfoEXT *drm_format_mod_info =
      vk_find_struct_const(pNext,
                           PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT);
   if (drm_format_mod_info)
      return drm_format_mod_info->drmFormatModifier;
   else
      return DRM_FORMAT_MOD_INVALID;
}

VKAPI_ATTR VkResult VKAPI_CALL
nvk_GetPhysicalDeviceImageFormatProperties2(
   VkPhysicalDevice physicalDevice,
   const VkPhysicalDeviceImageFormatInfo2 *pImageFormatInfo,
   VkImageFormatProperties2 *pImageFormatProperties)
{
   VK_FROM_HANDLE(nvk_physical_device, pdev, physicalDevice);

   const VkPhysicalDeviceExternalImageFormatInfo *external_info =
      vk_find_struct_const(pImageFormatInfo->pNext,
                           PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO);

   /* Initialize to zero in case we return VK_ERROR_FORMAT_NOT_SUPPORTED */
   memset(&pImageFormatProperties->imageFormatProperties, 0,
          sizeof(pImageFormatProperties->imageFormatProperties));

   uint64_t drm_format_mod =
      get_explicit_drm_format_mod(pImageFormatInfo->pNext);
   const struct vk_format_ycbcr_info *ycbcr_info =
      vk_format_get_ycbcr_info(pImageFormatInfo->format);

   /* For the purposes of these checks, we don't care about all the extra
    * YCbCr features and we just want the accumulation of features available
    * to all planes of the given format.
    */
   VkFormatFeatureFlags2 features;
   if (ycbcr_info == NULL) {
      features = nvk_get_image_plane_format_features(
         pdev, pImageFormatInfo->format, pImageFormatInfo->tiling,
         drm_format_mod);
   } else {
      features = ~0ull;
      assert(ycbcr_info->n_planes > 0);
      for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
         const VkFormat plane_format = ycbcr_info->planes[plane].format;
         features &= nvk_get_image_plane_format_features(
            pdev, plane_format, pImageFormatInfo->tiling, drm_format_mod);
      }
   }

   if (features == 0)
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR &&
       pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   if (ycbcr_info && pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   /* Maxwell B and earlier don't support sparse residency */
   if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
       pdev->info.cls_eng3d < MAXWELL_B)
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   /* Don't allow sparse on D32S8 cube maps.  The hardware doesn't seem to
    * handle these correctly and hard-faults instead of the expected soft
    * fault when there's sparse VA.
    */
   if (pImageFormatInfo->format == VK_FORMAT_D32_SFLOAT_S8_UINT &&
       (pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) &&
       (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
       pdev->info.cls_eng3d < TURING_A)
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   /* From the Vulkan 1.3.279 spec:
    *
    *    VUID-VkImageCreateInfo-tiling-04121
    *
    *    "If tiling is VK_IMAGE_TILING_LINEAR, flags must not contain
    *    VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
    *
    *    VUID-VkImageCreateInfo-imageType-00970
    *
    *    "If imageType is VK_IMAGE_TYPE_1D, flags must not contain
    *    VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
    */
   if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
       (pImageFormatInfo->type == VK_IMAGE_TYPE_1D ||
        pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR))
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   /* From the Vulkan 1.3.279 spec:
    *
    *    VUID-VkImageCreateInfo-flags-09403
    *
    *    "If flags contains VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT, flags
    *    must not include VK_IMAGE_CREATE_SPARSE_ALIASED_BIT,
    *    VK_IMAGE_CREATE_SPARSE_BINDING_BIT, or
    *    VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
    */
   if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT) &&
       (pImageFormatInfo->flags & (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT |
                                   VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
                                   VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   if (pImageFormatInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT &&
       pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   if ((pImageFormatInfo->usage & (VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_DECODE_SRC_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_ENCODE_DST_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)) &&
       (pImageFormatInfo->flags & (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT |
                                   VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   /* With disjoint, the client is allowed to create another image with a format
    * equal to the plane format and alias them. That only works if creating a
    * single plane image is equivalent to the given YCbCr plane. However, due
    * to video engine limitations, we have to create video YCbCr planes with
    * knowledge of all planes for block size calculations, so this won't work
    * since it wouldn't know the other planes.
    * See comment in nvk_image_init() below for more details.
    */
   if ((pImageFormatInfo->usage & (VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_DECODE_SRC_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_ENCODE_DST_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR |
                                   VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)) &&
       (pImageFormatInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT))
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   const uint32_t max_dim =
      nvk_image_max_dimension(&pdev->info, VK_IMAGE_TYPE_1D);
   VkExtent3D maxExtent;
   uint32_t maxArraySize;
   switch (pImageFormatInfo->type) {
   case VK_IMAGE_TYPE_1D:
      maxExtent = (VkExtent3D) { max_dim, 1, 1 };
      maxArraySize = 2048;
      break;
   case VK_IMAGE_TYPE_2D:
      maxExtent = (VkExtent3D) { max_dim, max_dim, 1 };
      maxArraySize = 2048;
      break;
   case VK_IMAGE_TYPE_3D:
      maxExtent = (VkExtent3D) { max_dim, max_dim, max_dim };
      maxArraySize = 1;
      break;
   default:
      UNREACHABLE("Invalid image type");
   }
   if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
      maxArraySize = 1;

   assert(util_is_power_of_two_nonzero(max_dim));
   uint32_t maxMipLevels = util_logbase2(max_dim) + 1;
   if (ycbcr_info != NULL || pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
       maxMipLevels = 1;

   if (pImageFormatInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
      maxArraySize = 1;
      maxMipLevels = 1;
   }

   VkSampleCountFlags sampleCounts = VK_SAMPLE_COUNT_1_BIT;
   if (pImageFormatInfo->tiling == VK_IMAGE_TILING_OPTIMAL &&
       pImageFormatInfo->type == VK_IMAGE_TYPE_2D &&
       ycbcr_info == NULL &&
       (features & (VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
                    VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
       !(pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)) {
      sampleCounts = VK_SAMPLE_COUNT_1_BIT |
                     VK_SAMPLE_COUNT_2_BIT |
                     VK_SAMPLE_COUNT_4_BIT |
                     VK_SAMPLE_COUNT_8_BIT;
   }

   /* From the Vulkan 1.2.199 spec:
    *
    *    "VK_IMAGE_CREATE_EXTENDED_USAGE_BIT specifies that the image can be
    *    created with usage flags that are not supported for the format the
    *    image is created with but are supported for at least one format a
    *    VkImageView created from the image can have."
    *
    * If VK_IMAGE_CREATE_EXTENDED_USAGE_BIT is set, views can be created with
    * different usage than the image so we can't always filter on usage.
    * There is one exception to this below for storage.
    */
   const VkImageUsageFlags image_usage = pImageFormatInfo->usage;
   VkImageUsageFlags view_usage = image_usage;
   if (pImageFormatInfo->flags & VK_IMAGE_CREATE_EXTENDED_USAGE_BIT)
      view_usage = 0;

   u_foreach_bit(b, view_usage) {
      VkFormatFeatureFlags2 usage_features =
         vk_image_usage_to_format_features(1 << b);
      if (usage_features && !(features & usage_features))
         return VK_ERROR_FORMAT_NOT_SUPPORTED;
   }

   const VkExternalMemoryProperties *ext_mem_props = NULL;
   if (external_info != NULL && external_info->handleType != 0) {
      bool tiling_has_explicit_layout;
      switch (pImageFormatInfo->tiling) {
      case VK_IMAGE_TILING_LINEAR:
      case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
         tiling_has_explicit_layout = true;
         break;
      case VK_IMAGE_TILING_OPTIMAL:
         tiling_has_explicit_layout = false;
         break;
      default:
         UNREACHABLE("Unsupported VkImageTiling");
      }

      switch (external_info->handleType) {
      case VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT:
         /* No special restrictions */
         if (tiling_has_explicit_layout) {
            /* With an explicit memory layout, we don't care which type of
             * fd the image belongs too. Both OPAQUE_FD and DMA_BUF are
             * interchangeable here.
             */
            ext_mem_props = &nvk_dma_buf_mem_props;
         } else {
            ext_mem_props = &nvk_opaque_fd_mem_props;
         }
         break;

      case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT:
         /* VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT requires
          * VK_IMAGE_TILING_LINEAR or VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT
          */
         if (!tiling_has_explicit_layout)
            return VK_ERROR_FORMAT_NOT_SUPPORTED;

         ext_mem_props = &nvk_dma_buf_mem_props;
         break;

      default:
         /* From the Vulkan 1.3.256 spec:
          *
          *    "If handleType is not compatible with the [parameters] in
          *    VkPhysicalDeviceImageFormatInfo2, then
          *    vkGetPhysicalDeviceImageFormatProperties2 returns
          *    VK_ERROR_FORMAT_NOT_SUPPORTED."
          */
         return vk_errorf(pdev, VK_ERROR_FORMAT_NOT_SUPPORTED,
                          "unsupported VkExternalMemoryHandleTypeFlagBits: %s ",
                           vk_ExternalMemoryHandleTypeFlagBits_to_str(external_info->handleType));
      }
   }

   unsigned plane_count = vk_format_get_plane_count(pImageFormatInfo->format);
   if (nvk_use_separate_zs(pdev, pImageFormatInfo->format))
      plane_count = 2;

   /* From the Vulkan 1.3.259 spec, VkImageCreateInfo:
    *
    *    VUID-VkImageCreateInfo-imageCreateFormatFeatures-02260
    *
    *    "If format is a multi-planar format, and if imageCreateFormatFeatures
    *    (as defined in Image Creation Limits) does not contain
    *    VK_FORMAT_FEATURE_DISJOINT_BIT, then flags must not contain
    *    VK_IMAGE_CREATE_DISJOINT_BIT"
    *
    * This is satisfied trivially because we support DISJOINT on all
    * multi-plane formats.  Also,
    *
    *    VUID-VkImageCreateInfo-format-01577
    *
    *    "If format is not a multi-planar format, and flags does not include
    *    VK_IMAGE_CREATE_ALIAS_BIT, flags must not contain
    *    VK_IMAGE_CREATE_DISJOINT_BIT"
    */
   if (plane_count == 1 &&
       !(pImageFormatInfo->flags & VK_IMAGE_CREATE_ALIAS_BIT) &&
       (pImageFormatInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT))
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   /* We don't support sparse residency for multi-plane images.  While we
    * could probably support sparse for VK_FORMAT_B8G8R8G8_422_UNORM, we
    * disable it because the standard block sizes are funky.
    */
   if ((plane_count > 1 || ycbcr_info != NULL) &&
       (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) &&
       (pImageFormatInfo->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT))
      return VK_ERROR_FORMAT_NOT_SUPPORTED;

   pImageFormatProperties->imageFormatProperties = (VkImageFormatProperties) {
      .maxExtent = maxExtent,
      .maxMipLevels = maxMipLevels,
      .maxArrayLayers = maxArraySize,
      .sampleCounts = sampleCounts,
      .maxResourceSize = UINT32_MAX, /* TODO */
   };

   vk_foreach_struct(s, pImageFormatProperties->pNext) {
      switch (s->sType) {
      case VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES: {
         VkExternalImageFormatProperties *p = (void *)s;
         /* From the Vulkan 1.3.256 spec:
          *
          *    "If handleType is 0, vkGetPhysicalDeviceImageFormatProperties2
          *    will behave as if VkPhysicalDeviceExternalImageFormatInfo was
          *    not present, and VkExternalImageFormatProperties will be
          *    ignored."
          *
          * This is true if and only if ext_mem_props == NULL
          */
         if (ext_mem_props != NULL)
            p->externalMemoryProperties = *ext_mem_props;
         break;
      }
      case VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_IMAGE_FORMAT_PROPERTIES: {
         VkSamplerYcbcrConversionImageFormatProperties *ycbcr_props = (void *) s;
         ycbcr_props->combinedImageSamplerDescriptorCount = plane_count;
         break;
      }
      case VK_STRUCTURE_TYPE_HOST_IMAGE_COPY_DEVICE_PERFORMANCE_QUERY_EXT: {
         VkHostImageCopyDevicePerformanceQueryEXT *host_props = (void *) s;
         host_props->optimalDeviceAccess = true;
         host_props->identicalMemoryLayout = true;
         break;
      }
      default:
         vk_debug_ignored_stype(s->sType);
         break;
      }
   }

   return VK_SUCCESS;
}

static enum nil_image_dim
vk_image_type_to_nil_dim(VkImageType type)
{
   switch (type) {
   case VK_IMAGE_TYPE_1D:  return NIL_IMAGE_DIM_1D;
   case VK_IMAGE_TYPE_2D:  return NIL_IMAGE_DIM_2D;
   case VK_IMAGE_TYPE_3D:  return NIL_IMAGE_DIM_3D;
   default:
      UNREACHABLE("Invalid image type");
   }
}

static VkSparseImageFormatProperties
nvk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects,
                                const enum pipe_format format,
                                const enum nil_image_dim dim,
                                const enum nil_sample_layout sample_layout)
{
   struct nil_Extent4D_Pixels sparse_block_extent_px =
      nil_sparse_block_extent_px(nil_format(format), dim, sample_layout);

   assert(sparse_block_extent_px.array_len == 1);

   VkSparseImageFormatProperties sparse_format_props = {
      .aspectMask = aspects,
      .flags = VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT,
      .imageGranularity = {
         .width = sparse_block_extent_px.width,
         .height = sparse_block_extent_px.height,
         .depth = sparse_block_extent_px.depth,
      },
   };

   return sparse_format_props;
}

VKAPI_ATTR void VKAPI_CALL
nvk_GetPhysicalDeviceSparseImageFormatProperties2(
    VkPhysicalDevice physicalDevice,
    const VkPhysicalDeviceSparseImageFormatInfo2* pFormatInfo,
    uint32_t *pPropertyCount,
    VkSparseImageFormatProperties2 *pProperties)
{
   VkResult result;

   /* Check if the given format info is valid first before returning sparse
    * props.  The easiest way to do this is to just call
    * nvk_GetPhysicalDeviceImageFormatProperties2()
    */
   const VkPhysicalDeviceImageFormatInfo2 img_fmt_info = {
      .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
      .format = pFormatInfo->format,
      .type = pFormatInfo->type,
      .tiling = pFormatInfo->tiling,
      .usage = pFormatInfo->usage,
      .flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
               VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT,
   };

   VkImageFormatProperties2 img_fmt_props2 = {
      .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
      .pNext = NULL,
   };

   result = nvk_GetPhysicalDeviceImageFormatProperties2(physicalDevice,
                                                        &img_fmt_info,
                                                        &img_fmt_props2);
   if (result != VK_SUCCESS) {
      *pPropertyCount = 0;
      return;
   }

   const VkImageFormatProperties *props = &img_fmt_props2.imageFormatProperties;
   if (!(pFormatInfo->samples & props->sampleCounts)) {
      *pPropertyCount = 0;
      return;
   }

   VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out,
                          pProperties, pPropertyCount);

   VkImageAspectFlags aspects = vk_format_aspects(pFormatInfo->format);
   const enum pipe_format pipe_format =
      nvk_format_to_pipe_format(pFormatInfo->format);
   const enum nil_image_dim dim = vk_image_type_to_nil_dim(pFormatInfo->type);
   const enum nil_sample_layout sample_layout =
      nil_choose_sample_layout(pFormatInfo->samples);

   vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, props) {
      props->properties = nvk_fill_sparse_image_fmt_props(aspects, pipe_format,
                                                          dim, sample_layout);
   }
}

/* To use compression and larger page sizes, we need to signal to the kernel
 * that the memory requested is going to be VRAM resident. However, this
 * comes with an issue where said memory can't be evicted to host RAM under
 * pressure, so we work around this by going with a dedicated allocation for
 * color, Z/S, and storage image targets which are the main types that would
 * benefit from compression as they're heavy on writes. Additionally, they
 * also aren't the majority of memory used, so they can be safely pinned in
 * VRAM without worrying about eviction under high pressure.
 *
 * There are some additional restrictions we need to keep in mind, however:
 * 1. We can only enable this for Turing onwards because prior architectures
 *    relied on firmware to manage the compression tags, and it's impossible to
 *    do this on nouveau. Additionally, since compression needs kernel changes,
 *    we can only enable it if the detected kernel supports it.
 *
 * 2. Given our approach depends on dedicated allocations, we can't enable
 *    compression for sparse images as dedicated allocations are not compatible
 *    with sparse.
 *
 * 3. In similar vein, we currently don't do multiplanar dedicated allocations
 *    so we can't do compression for multi-plane YCbCr images.
 *
 * 4. Host copies are a complete no-go for compression as the host doesn't know
 *    about the modified data layout nor the compression tags.
 *
 * 5. The API for VK_EXT_image_drm_format_modifier requires that we report the
 *    supported modifiers in GetPhysicalDeviceFormatProperties2(). However,
 *    since we can only know whether an image is compressed or not at bind time
 *    we can't actually expose any of the compressed modifiers in case the app
 *    chooses a compressed modifier for a non-compressed image. So for now, we
 *    have to disable compression for TILING_DRM_FORMAT_MODIFIER_EXT images.
 *
 * This helper enforces these restrictions and also makes sure to enable
 * compression for storage, color, and Z/S targets only so as to avoid pinning
 * too many things to VRAM.
 */
static bool
nvk_image_can_compress(const struct nvkmd_pdev *nvkmd_pdev,
                       const struct nvk_image *image)
{
   if (nvkmd_pdev->kmd_info.has_compression) {
      if (image->plane_count > 1 ||
          image->vk.usage & (VK_IMAGE_USAGE_HOST_TRANSFER_BIT) ||
          image->vk.create_flags & (VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
                                    VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
         return false;
      else if (image->vk.usage & (VK_IMAGE_USAGE_STORAGE_BIT |
                                  VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
                                  VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT) &&
               image->vk.tiling == VK_IMAGE_TILING_OPTIMAL)
         return true;
      else
         return false;
   } else
      return false;
}

static VkResult
nvk_image_init(struct nvk_device *dev,
               struct nvk_image *image,
               const VkImageCreateInfo *pCreateInfo)
{
   const struct nvk_physical_device *pdev = nvk_device_physical(dev);

   vk_image_init(&dev->vk, &image->vk, pCreateInfo);

   nil_image_usage_flags usage = 0;
   if (image->vk.tiling == VK_IMAGE_TILING_LINEAR)
      usage |= NIL_IMAGE_USAGE_LINEAR_BIT;
   if (image->vk.create_flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT)
      usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;
   if (image->vk.create_flags & VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT)
      usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;

   /* In order to be able to clear 3D depth/stencil images, we need to bind
    * them as 2D arrays.  Fortunately, 3D depth/stencil shouldn't be common.
    */
   if ((image->vk.aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
                             VK_IMAGE_ASPECT_STENCIL_BIT)) &&
       image->vk.image_type == VK_IMAGE_TYPE_3D)
      usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;

   image->plane_count = vk_format_get_plane_count(image->vk.format);
   image->disjoint = image->plane_count > 1 &&
                     (image->vk.create_flags & VK_IMAGE_CREATE_DISJOINT_BIT);

   if (nvk_use_separate_zs(pdev, image->vk.format)) {
      image->separate_zs = true;
      image->plane_count = 2;
   }

   if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
      /* Sparse multiplane is not supported */
      assert(image->plane_count == 1);
      usage |= NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT;
   }

   if (image->vk.usage & (VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
                          VK_IMAGE_USAGE_VIDEO_DECODE_SRC_BIT_KHR |
                          VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
                          VK_IMAGE_USAGE_VIDEO_ENCODE_DST_BIT_KHR |
                          VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR |
                          VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR))
      usage |= NIL_IMAGE_USAGE_VIDEO_BIT;

   /* We set compression on VkImage creation in order to be able to signal to
    * NIL that the image will be compressed which would let NIL choose the
    * appropriate PTE kinds, and also to mark the VkImage as compressed so that
    * in GetImageMemoryRequirements() we are able to detect it and specify that
    * we prefer a dedicated allocation for it.
    */
   image->can_compress = nvk_image_can_compress(dev->nvkmd->pdev, image);
   if (!image->can_compress)
      usage |= NIL_IMAGE_USAGE_UNCOMPRESSED_BIT;

   uint32_t explicit_row_stride_B = 0;

   /* This section is removed by the optimizer for non-ANDROID builds */
   if (vk_image_is_android_native_buffer(&image->vk)) {
      VkImageDrmFormatModifierExplicitCreateInfoEXT eci;
      VkSubresourceLayout a_plane_layouts[4];
      VkResult result = vk_android_get_anb_layout(
         pCreateInfo, &eci, a_plane_layouts, 4);
      if (result != VK_SUCCESS)
         return result;

      image->vk.drm_format_mod = eci.drmFormatModifier;
      explicit_row_stride_B = eci.pPlaneLayouts[0].rowPitch;
   }

   uint32_t max_alignment_B = 0;
   const VkImageAlignmentControlCreateInfoMESA *alignment =
      vk_find_struct_const(pCreateInfo->pNext,
                           IMAGE_ALIGNMENT_CONTROL_CREATE_INFO_MESA);
   if (alignment && alignment->maximumRequestedAlignment) {
      assert(util_is_power_of_two_or_zero(alignment->maximumRequestedAlignment));
      max_alignment_B = alignment->maximumRequestedAlignment;
   }

   if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
      /* Modifiers are not supported with YCbCr */
      assert(image->plane_count == 1);

      const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_explicit_info =
         vk_find_struct_const(pCreateInfo->pNext,
                              IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
      if (mod_explicit_info) {
         image->vk.drm_format_mod = mod_explicit_info->drmFormatModifier;
         /* Normally with explicit modifiers, the client specifies all strides,
          * however in our case, we can only really make use of this in the linear
          * case, and we can only create 2D non-array linear images, so ultimately
          * we only care about the row stride. 
          */
         explicit_row_stride_B = mod_explicit_info->pPlaneLayouts->rowPitch;
      } else {
         const struct VkImageDrmFormatModifierListCreateInfoEXT *mod_list_info =
            vk_find_struct_const(pCreateInfo->pNext,
                                 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);

         enum pipe_format p_format =
            nvk_format_to_pipe_format(image->vk.format);
         image->vk.drm_format_mod =
            nil_select_best_drm_format_mod(&pdev->info, nil_format(p_format),
                                           mod_list_info->drmFormatModifierCount,
                                           mod_list_info->pDrmFormatModifiers);
         assert(image->vk.drm_format_mod != DRM_FORMAT_MOD_INVALID);
      }

      if (image->vk.drm_format_mod == DRM_FORMAT_MOD_LINEAR) {
         /* We only have one shadow plane per nvk_image */
         assert(image->plane_count == 1);

         struct nil_image_init_info tiled_shadow_nil_info = {
            .dim = vk_image_type_to_nil_dim(image->vk.image_type),
            .format = nil_format(nvk_format_to_pipe_format(image->vk.format)),
            .modifier = DRM_FORMAT_MOD_INVALID,
            .extent_px = {
               .width = image->vk.extent.width,
               .height = image->vk.extent.height,
               .depth = image->vk.extent.depth,
               .array_len = image->vk.array_layers,
            },
            .levels = image->vk.mip_levels,
            .samples = image->vk.samples,
            .usage = usage & ~NIL_IMAGE_USAGE_LINEAR_BIT,
         };
         bool ok = nil_image_init(&pdev->info,
                                  &image->linear_tiled_shadow.nil,
                                  &tiled_shadow_nil_info);
         if (!ok)
            return vk_errorf(dev, VK_ERROR_UNKNOWN,
                             "Invalid image creation parameters");
      }
   }

   /* The video decode engine needs the block size to be the same across chroma
    * and luma planes, so in order to work around this limitation we gather all
    * the info for NIL early, which would give it enough information to get and
    * use the smallest block size for all planes.
    */
   const struct vk_format_ycbcr_info *ycbcr_info =
      vk_format_get_ycbcr_info(image->vk.format);
   struct nil_image_init_info nil_info[NVK_MAX_IMAGE_PLANES];
   for (uint8_t plane = 0; plane < image->plane_count; plane++) {
      VkFormat format = ycbcr_info ?
         ycbcr_info->planes[plane].format : image->vk.format;
      const uint8_t width_scale = ycbcr_info ?
         ycbcr_info->planes[plane].denominator_scales[0] : 1;
      const uint8_t height_scale = ycbcr_info ?
         ycbcr_info->planes[plane].denominator_scales[1] : 1;

      if (image->separate_zs) {
         if (plane == 0)
            format = vk_format_depth_only(format);
         else if (plane == 1)
            format = vk_format_stencil_only(format);
      }

      nil_info[plane] = (struct nil_image_init_info) {
         .dim = vk_image_type_to_nil_dim(image->vk.image_type),
         .format = nil_format(nvk_format_to_pipe_format(format)),
         .modifier = image->vk.drm_format_mod,
         .extent_px = {
            .width = image->vk.extent.width / width_scale,
            .height = image->vk.extent.height / height_scale,
            .depth = image->vk.extent.depth,
            .array_len = image->vk.array_layers,
         },
         .levels = image->vk.mip_levels,
         .samples = image->vk.samples,
         .usage = usage,
         .explicit_row_stride_B = explicit_row_stride_B,
         .max_alignment_B = max_alignment_B,
      };
   }

   if (usage & NIL_IMAGE_USAGE_VIDEO_BIT) {
      assert(!image->disjoint);
      for (uint8_t plane = 0; plane < image->plane_count; plane++) {
         bool ok = nil_image_init_planar(&pdev->info,
                                         &image->planes[plane].nil,
                                         nil_info, plane,
                                         image->plane_count);
         if (!ok)
            return vk_errorf(dev, VK_ERROR_UNKNOWN,
                             "Invalid image creation parameters");
      }
   } else {
      for (uint8_t plane = 0; plane < image->plane_count; plane++) {
         bool ok = nil_image_init(&pdev->info,
                                  &image->planes[plane].nil,
                                  &nil_info[plane]);
         if (!ok)
            return vk_errorf(dev, VK_ERROR_UNKNOWN,
                             "Invalid image creation parameters");
      }
   }

   const enum pipe_format plane0_format = image->planes[0].nil.format.p_format;
   if (plane0_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
      struct nil_image_init_info stencil_nil_info = {
         .dim = vk_image_type_to_nil_dim(image->vk.image_type),
         .format = nil_format(PIPE_FORMAT_R32_UINT),
         .modifier = DRM_FORMAT_MOD_INVALID,
         .extent_px = {
            .width = image->vk.extent.width,
            .height = image->vk.extent.height,
            .depth = image->vk.extent.depth,
            .array_len = image->vk.array_layers,
         },
         .levels = image->vk.mip_levels,
         .samples = image->vk.samples,
         .usage = usage,
      };

      bool ok = nil_image_init(&pdev->info,
                               &image->stencil_copy_temp.nil,
                               &stencil_nil_info);
      if (!ok)
         return vk_errorf(dev, VK_ERROR_UNKNOWN,
                          "Invalid image creation parameters");
   }

   return VK_SUCCESS;
}

static void
nvk_image_plane_size_align_B(struct nvk_device *dev,
                             const struct nvk_image *image,
                             const struct nvk_image_plane *plane,
                             uint64_t *size_B_out, uint64_t *align_B_out)
{
   const struct nvk_physical_device *pdev = nvk_device_physical(dev);
   const bool sparse_bound =
      image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT;

   assert(util_is_power_of_two_or_zero64(plane->nil.align_B));
   if (sparse_bound || plane->nil.pte_kind) {
      *align_B_out = MAX2(plane->nil.align_B, pdev->nvkmd->bind_align_B);
   } else {
      *align_B_out = plane->nil.align_B;
   }
   *size_B_out = align64(plane->nil.size_B, *align_B_out);
}

static VkResult
nvk_image_plane_alloc_va(struct nvk_device *dev,
                         const struct nvk_image *image,
                         struct nvk_image_plane *plane)
{
   VkResult result;

   const bool sparse_bound =
      image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT;
   const bool sparse_resident =
      image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
   assert(sparse_bound || !sparse_resident);

   if (sparse_bound || plane->nil.pte_kind) {
      enum nvkmd_va_flags va_flags = 0;
      if (sparse_resident)
         va_flags |= NVKMD_VA_SPARSE;

      uint64_t va_size_B, va_align_B;
      nvk_image_plane_size_align_B(dev, image, plane, &va_size_B, &va_align_B);

      result = nvkmd_dev_alloc_va(dev->nvkmd, &dev->vk.base,
                                  va_flags, plane->nil.pte_kind,
                                  va_size_B, va_align_B,
                                  0 /* fixed_addr */, &plane->va);
      if (result != VK_SUCCESS)
         return result;

      plane->addr = plane->va->addr;
   }

   return VK_SUCCESS;
}

static void
nvk_image_plane_finish(struct nvk_device *dev,
                       struct nvk_image_plane *plane,
                       VkImageCreateFlags create_flags,
                       const VkAllocationCallbacks *pAllocator)
{
   if (plane->va != NULL)
      nvkmd_va_free(plane->va);
}

static void
nvk_image_finish(struct nvk_device *dev, struct nvk_image *image,
                 const VkAllocationCallbacks *pAllocator)
{
   for (uint8_t plane = 0; plane < image->plane_count; plane++) {
      nvk_image_plane_finish(dev, &image->planes[plane],
                             image->vk.create_flags, pAllocator);
   }

   if (image->stencil_copy_temp.nil.size_B > 0) {
      nvk_image_plane_finish(dev, &image->stencil_copy_temp,
                             image->vk.create_flags, pAllocator);
   }

   assert(image->linear_tiled_shadow.va == NULL);
   if (image->linear_tiled_shadow_mem != NULL)
      nvkmd_mem_unref(image->linear_tiled_shadow_mem);

   vk_image_finish(&image->vk);
}

VKAPI_ATTR VkResult VKAPI_CALL
nvk_CreateImage(VkDevice _device,
                const VkImageCreateInfo *pCreateInfo,
                const VkAllocationCallbacks *pAllocator,
                VkImage *pImage)
{
   VK_FROM_HANDLE(nvk_device, dev, _device);
   UNUSED const struct nvk_physical_device *pdev = nvk_device_physical(dev);
   struct nvk_image *image;
   VkResult result;

   if (wsi_common_is_swapchain_image(pCreateInfo)) {
      return wsi_common_create_swapchain_image(&pdev->wsi_device,
                                               pCreateInfo,
                                               pImage);
   }

   image = vk_zalloc2(&dev->vk.alloc, pAllocator, sizeof(*image), 8,
                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
   if (!image)
      return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);

   result = nvk_image_init(dev, image, pCreateInfo);
   if (result != VK_SUCCESS) {
      vk_free2(&dev->vk.alloc, pAllocator, image);
      return result;
   }

   if (image->vk.create_flags & (VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
                                 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)) {
      for (uint8_t plane = 0; plane < image->plane_count; plane++) {
         result = nvk_image_plane_alloc_va(dev, image, &image->planes[plane]);
         if (result != VK_SUCCESS)
            goto fail;
      }

      if (image->stencil_copy_temp.nil.size_B > 0) {
         result = nvk_image_plane_alloc_va(dev, image, &image->stencil_copy_temp);
         if (result != VK_SUCCESS)
            goto fail;
      }
   }

   if (image->linear_tiled_shadow.nil.size_B > 0) {
      struct nvk_image_plane *shadow = &image->linear_tiled_shadow;
      result = nvkmd_dev_alloc_tiled_mem(dev->nvkmd, &dev->vk.base,
                                         shadow->nil.size_B, shadow->nil.align_B,
                                         shadow->nil.pte_kind, shadow->nil.tile_mode,
                                         NVKMD_MEM_LOCAL,
                                         &image->linear_tiled_shadow_mem);
      if (result != VK_SUCCESS)
         goto fail;

      shadow->addr = image->linear_tiled_shadow_mem->va->addr;
   }

   /* This section is removed by the optimizer for non-ANDROID builds */
   if (vk_image_is_android_native_buffer(&image->vk)) {
      result = vk_android_import_anb(&dev->vk, pCreateInfo, pAllocator,
                                     &image->vk);
      if (result != VK_SUCCESS)
         goto fail;
   }

   *pImage = nvk_image_to_handle(image);

   return VK_SUCCESS;

fail:
   nvk_image_finish(dev, image, pAllocator);
   vk_free2(&dev->vk.alloc, pAllocator, image);
   return result;
}

VKAPI_ATTR void VKAPI_CALL
nvk_DestroyImage(VkDevice device,
                 VkImage _image,
                 const VkAllocationCallbacks *pAllocator)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   VK_FROM_HANDLE(nvk_image, image, _image);

   if (!image)
      return;

   nvk_image_finish(dev, image, pAllocator);
   vk_free2(&dev->vk.alloc, pAllocator, image);
}

static void
nvk_image_plane_add_req(struct nvk_device *dev,
                        const struct nvk_image *image,
                        const struct nvk_image_plane *plane,
                        uint64_t *size_B, uint32_t *align_B)
{
   assert(util_is_power_of_two_or_zero64(*align_B));
   uint64_t plane_size_B, plane_align_B;
   nvk_image_plane_size_align_B(dev, image, plane,
                                &plane_size_B, &plane_align_B);

   *align_B = MAX2(*align_B, plane_align_B);
   *size_B = align64(*size_B, plane_align_B);
   *size_B += plane_size_B;
}

static void
nvk_get_image_memory_requirements(struct nvk_device *dev,
                                  struct nvk_image *image,
                                  VkImageAspectFlags aspects,
                                  VkMemoryRequirements2 *pMemoryRequirements)
{
   const struct nvk_physical_device *pdev = nvk_device_physical(dev);
   uint32_t memory_types = (1 << pdev->mem_type_count) - 1;

   /* Remove non host visible heaps from the types for host image copy in case
    * of potential issues when we do not have ReBAR.
    */
   if (pdev->info.bar_size_B < pdev->info.vram_size_B &&
       image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
      for (uint32_t i = 0; i < pdev->mem_type_count; i++) {
         if (!(pdev->mem_types[i].propertyFlags &
             VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT))
            memory_types &= ~BITFIELD_BIT(i);
      }
   }

   uint64_t size_B = 0;
   uint32_t align_B = 0;
   if (image->disjoint) {
      uint8_t plane = nvk_image_memory_aspects_to_plane(image, aspects);
      nvk_image_plane_add_req(dev, image, &image->planes[plane],
                              &size_B, &align_B);
   } else {
      for (unsigned plane = 0; plane < image->plane_count; plane++) {
         nvk_image_plane_add_req(dev, image, &image->planes[plane],
                                 &size_B, &align_B);
      }
   }

   if (image->stencil_copy_temp.nil.size_B > 0) {
      nvk_image_plane_add_req(dev, image, &image->stencil_copy_temp,
                              &size_B, &align_B);
   }

   pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types;
   pMemoryRequirements->memoryRequirements.alignment = align_B;
   pMemoryRequirements->memoryRequirements.size = size_B;

   vk_foreach_struct_const(ext, pMemoryRequirements->pNext) {
      switch (ext->sType) {
      case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
         VkMemoryDedicatedRequirements *dedicated = (void *)ext;
         if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
            dedicated->prefersDedicatedAllocation = true;
            dedicated->requiresDedicatedAllocation = true;
         } else if (image->can_compress) {
            /* We need dedicated allocations as compressed images have to be
             * pinned to VRAM due to nouveau, and we can't have a separate
             * memory type that's pinned and non evictable due to the Vulkan API
             * disallowing equivalent image properties returning different
             * memory types. We aren't allowed to require dedicated allocations
             * but we can signal that we prefer them.
             */
            dedicated->prefersDedicatedAllocation = true;
            dedicated->requiresDedicatedAllocation = false;
         } else {
            dedicated->prefersDedicatedAllocation = false;
            dedicated->requiresDedicatedAllocation = false;
         }
         break;
      }
      default:
         vk_debug_ignored_stype(ext->sType);
         break;
      }
   }
}

VKAPI_ATTR void VKAPI_CALL
nvk_GetImageMemoryRequirements2(VkDevice device,
                                const VkImageMemoryRequirementsInfo2 *pInfo,
                                VkMemoryRequirements2 *pMemoryRequirements)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   VK_FROM_HANDLE(nvk_image, image, pInfo->image);

   const VkImagePlaneMemoryRequirementsInfo *plane_info =
      vk_find_struct_const(pInfo->pNext, IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO);
   const VkImageAspectFlags aspects =
      image->disjoint ? plane_info->planeAspect : image->vk.aspects;

   nvk_get_image_memory_requirements(dev, image, aspects,
                                     pMemoryRequirements);
}

VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageMemoryRequirements(VkDevice device,
                                     const VkDeviceImageMemoryRequirements *pInfo,
                                     VkMemoryRequirements2 *pMemoryRequirements)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   ASSERTED VkResult result;
   struct nvk_image image = {0};

   result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
   assert(result == VK_SUCCESS);

   const VkImageAspectFlags aspects =
      image.disjoint ? pInfo->planeAspect : image.vk.aspects;

   nvk_get_image_memory_requirements(dev, &image, aspects,
                                     pMemoryRequirements);

   nvk_image_finish(dev, &image, NULL);
}

static VkSparseImageMemoryRequirements
nvk_fill_sparse_image_memory_reqs(const struct nil_image *nil,
                                  const struct nil_image *stencil_tmp,
                                  VkImageAspectFlags aspects)
{
   VkSparseImageFormatProperties sparse_format_props =
      nvk_fill_sparse_image_fmt_props(aspects, nil->format.p_format,
                                      nil->dim, nil->sample_layout);

   assert(nil->mip_tail_first_lod <= nil->num_levels);
   VkSparseImageMemoryRequirements sparse_memory_reqs = {
      .formatProperties = sparse_format_props,
      .imageMipTailFirstLod = nil->mip_tail_first_lod,
      .imageMipTailStride = 0,
   };

   if (nil->mip_tail_first_lod == 0) {
      sparse_memory_reqs.imageMipTailSize = nil->size_B;
      sparse_memory_reqs.imageMipTailOffset = 0;
   } else if (nil->mip_tail_first_lod < nil->num_levels) {
      sparse_memory_reqs.imageMipTailSize =
         nil_image_mip_tail_size_B(nil) * nil->extent_px.array_len;
      sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET;
   } else {
      sparse_memory_reqs.imageMipTailSize = 0;
      sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET;
   }

   if (stencil_tmp != NULL)
      sparse_memory_reqs.imageMipTailSize += stencil_tmp->size_B;

   return sparse_memory_reqs;
}

static void
nvk_get_image_sparse_memory_requirements(
   struct nvk_device *dev,
   struct nvk_image *image,
   VkImageAspectFlags aspects,
   uint32_t *pSparseMemoryRequirementCount,
   VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
{
   VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out,
                          pSparseMemoryRequirements,
                          pSparseMemoryRequirementCount);

   /* From the Vulkan 1.3.279 spec:
    *
    *    "The sparse image must have been created using the
    *    VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse
    *    image memory requirements."
    */
   if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
      return;

   /* We don't support multiplane sparse for now */
   if (image->plane_count > 1)
      return;

   const struct nil_image *stencil_tmp = NULL;
   if (image->stencil_copy_temp.nil.size_B > 0)
      stencil_tmp = &image->stencil_copy_temp.nil;

   vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, reqs) {
      reqs->memoryRequirements =
         nvk_fill_sparse_image_memory_reqs(&image->planes[0].nil,
                                           stencil_tmp, aspects);
   };
}

VKAPI_ATTR void VKAPI_CALL
nvk_GetImageSparseMemoryRequirements2(
   VkDevice device,
   const VkImageSparseMemoryRequirementsInfo2* pInfo,
   uint32_t* pSparseMemoryRequirementCount,
   VkSparseImageMemoryRequirements2* pSparseMemoryRequirements)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   VK_FROM_HANDLE(nvk_image, image, pInfo->image);

   const VkImageAspectFlags aspects = image->vk.aspects;

   nvk_get_image_sparse_memory_requirements(dev, image, aspects,
                                            pSparseMemoryRequirementCount,
                                            pSparseMemoryRequirements);
}

VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageSparseMemoryRequirements(
   VkDevice device,
   const VkDeviceImageMemoryRequirements* pInfo,
   uint32_t *pSparseMemoryRequirementCount,
   VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   ASSERTED VkResult result;
   struct nvk_image image = {0};

   result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
   assert(result == VK_SUCCESS);

   const VkImageAspectFlags aspects =
      image.disjoint ? pInfo->planeAspect : image.vk.aspects;

   nvk_get_image_sparse_memory_requirements(dev, &image, aspects,
                                            pSparseMemoryRequirementCount,
                                            pSparseMemoryRequirements);

   nvk_image_finish(dev, &image, NULL);
}

static void
nvk_get_image_subresource_layout(struct nvk_device *dev,
                                 struct nvk_image *image,
                                 const VkImageSubresource2KHR *pSubresource,
                                 VkSubresourceLayout2KHR *pLayout)
{
   const VkImageSubresource *isr = &pSubresource->imageSubresource;

   const uint8_t p = nvk_image_memory_aspects_to_plane(image, isr->aspectMask);
   const struct nvk_image_plane *plane = &image->planes[p];

   uint64_t offset_B = 0;
   if (!image->disjoint) {
      uint32_t align_B = 0;
      for (unsigned plane = 0; plane < p; plane++) {
         nvk_image_plane_add_req(dev, image, &image->planes[plane],
                                 &offset_B, &align_B);
      }
   }
   offset_B += nil_image_level_layer_offset_B(&plane->nil, isr->mipLevel,
                                              isr->arrayLayer);

   VkSubresourceHostMemcpySizeEXT *host_memcpy_size =
      vk_find_struct(pLayout->pNext, SUBRESOURCE_HOST_MEMCPY_SIZE_EXT);
   if (host_memcpy_size) {
      host_memcpy_size->size =
         nil_image_level_layer_size_B(&plane->nil, isr->mipLevel) *
         plane->nil.extent_px.array_len;
   }

   pLayout->subresourceLayout = (VkSubresourceLayout) {
      .offset = offset_B,
      .size = nil_image_level_size_B(&plane->nil, isr->mipLevel),
      .rowPitch = plane->nil.levels[isr->mipLevel].row_stride_B,
      .arrayPitch = plane->nil.array_stride_B,
      .depthPitch = nil_image_level_depth_stride_B(&plane->nil, isr->mipLevel),
   };
}

VKAPI_ATTR void VKAPI_CALL
nvk_GetImageSubresourceLayout2KHR(VkDevice device,
                                  VkImage _image,
                                  const VkImageSubresource2KHR *pSubresource,
                                  VkSubresourceLayout2KHR *pLayout)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   VK_FROM_HANDLE(nvk_image, image, _image);

   nvk_get_image_subresource_layout(dev, image, pSubresource, pLayout);
}

VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageSubresourceLayoutKHR(
    VkDevice device,
    const VkDeviceImageSubresourceInfoKHR *pInfo,
    VkSubresourceLayout2KHR *pLayout)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   ASSERTED VkResult result;
   struct nvk_image image = {0};

   result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
   assert(result == VK_SUCCESS);

   nvk_get_image_subresource_layout(dev, &image, pInfo->pSubresource, pLayout);

   nvk_image_finish(dev, &image, NULL);
}

static VkResult
nvk_image_plane_bind(struct nvk_device *dev,
                     struct nvk_image *image,
                     struct nvk_image_plane *plane,
                     struct nvk_device_memory *mem,
                     uint64_t *offset_B)
{
   uint64_t plane_size_B, plane_align_B;
   nvk_image_plane_size_align_B(dev, image, plane,
                                &plane_size_B, &plane_align_B);
   *offset_B = align64(*offset_B, plane_align_B);

   const bool not_shared = !(mem->mem->flags & NVKMD_MEM_SHARED);

   if (plane->nil.pte_kind != 0) {
      if (mem->dedicated_image == image && image->can_compress && not_shared) {
         image->is_compressed = true;
         plane->addr = mem->mem->va->addr + *offset_B;
      } else {
         VkResult result = nvk_image_plane_alloc_va(dev, image, plane);
         if (result != VK_SUCCESS)
            return result;
         result = nvkmd_va_bind_mem(plane->va, &image->vk.base, 0,
                                    mem->mem, *offset_B,
                                    plane->va->size_B);
         if (result != VK_SUCCESS)
            return result;
      }
   } else {
      plane->addr = mem->mem->va->addr + *offset_B;
   }

   if (image->vk.usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) {
      plane->host_mem = mem;
      plane->host_offset = *offset_B;
   }

   *offset_B += plane_size_B;

   return VK_SUCCESS;
}

static VkResult
nvk_bind_image_memory(struct nvk_device *dev,
                      const VkBindImageMemoryInfo *info)
{
   VK_FROM_HANDLE(nvk_device_memory, mem, info->memory);
   VK_FROM_HANDLE(nvk_image, image, info->image);
   uint64_t offset_B = info->memoryOffset;
   VkResult result;

#if DETECT_OS_ANDROID
   const VkNativeBufferANDROID *anb_info =
      vk_find_struct_const(info->pNext, NATIVE_BUFFER_ANDROID);
   if (anb_info != NULL && anb_info->handle != NULL) {
      /* We do the actual bind the end of CreateImage() */
      assert(mem == NULL);
      return VK_SUCCESS;
   }
#endif

   /* Ignore this struct on Android, we cannot access swapchain structures there. */
#ifdef NVK_USE_WSI_PLATFORM
   if (mem == NULL) {
      const VkBindImageMemorySwapchainInfoKHR *swapchain_info =
         vk_find_struct_const(info->pNext, BIND_IMAGE_MEMORY_SWAPCHAIN_INFO_KHR);
      assert(swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE);
      mem = nvk_device_memory_from_handle(
         wsi_common_get_memory(swapchain_info->swapchain, swapchain_info->imageIndex));
      offset_B = 0;
   }
#endif

   assert(mem != NULL);
   if (image->disjoint) {
      const VkBindImagePlaneMemoryInfo *plane_info =
         vk_find_struct_const(info->pNext, BIND_IMAGE_PLANE_MEMORY_INFO);
      const uint8_t plane =
         nvk_image_memory_aspects_to_plane(image, plane_info->planeAspect);
      result = nvk_image_plane_bind(dev, image, &image->planes[plane],
                                    mem, &offset_B);
      if (result != VK_SUCCESS)
         return result;
   } else {
      for (unsigned plane = 0; plane < image->plane_count; plane++) {
         result = nvk_image_plane_bind(dev, image, &image->planes[plane],
                                       mem, &offset_B);
         if (result != VK_SUCCESS)
            return result;
      }
   }

   if (image->stencil_copy_temp.nil.size_B > 0) {
      result = nvk_image_plane_bind(dev, image, &image->stencil_copy_temp,
                                    mem, &offset_B);
      if (result != VK_SUCCESS)
         return result;
   }

   return VK_SUCCESS;
}

VKAPI_ATTR VkResult VKAPI_CALL
nvk_BindImageMemory2(VkDevice device,
                     uint32_t bindInfoCount,
                     const VkBindImageMemoryInfo *pBindInfos)
{
   VK_FROM_HANDLE(nvk_device, dev, device);
   VkResult first_error_or_success = VK_SUCCESS;

   for (uint32_t i = 0; i < bindInfoCount; ++i) {
      VkResult result = nvk_bind_image_memory(dev, &pBindInfos[i]);

      const VkBindMemoryStatusKHR *status =
         vk_find_struct_const(pBindInfos[i].pNext, BIND_MEMORY_STATUS_KHR);
      if (status != NULL && status->pResult != NULL)
         *status->pResult = result;

      if (first_error_or_success == VK_SUCCESS)
         first_error_or_success = result;
   }

   return first_error_or_success;
}


static VkResult
queue_image_plane_bind(struct nvk_queue *queue,
                       const struct nvk_image_plane *plane,
                       const VkSparseImageMemoryBind *bind)
{
   VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
   uint64_t image_bind_offset_B;

   const uint64_t mem_bind_offset_B = bind->memoryOffset;
   const uint32_t layer = bind->subresource.arrayLayer;
   const uint32_t level = bind->subresource.mipLevel;

   const struct nil_tiling plane_tiling = plane->nil.levels[level].tiling;
   const uint32_t tile_size_B = nil_tiling_size_B(&plane_tiling);

   const struct nil_Extent4D_Pixels bind_extent_px = {
      .width = bind->extent.width,
      .height = bind->extent.height,
      .depth = bind->extent.depth,
      .array_len = 1,
   };
   const struct nil_Offset4D_Pixels bind_offset_px = {
      .x = bind->offset.x,
      .y = bind->offset.y,
      .z = bind->offset.z,
      .a = layer,
   };

   const struct nil_Extent4D_Pixels level_extent_px =
      nil_image_level_extent_px(&plane->nil, level);
   const struct nil_Extent4D_Tiles level_extent_tl =
      nil_extent4d_px_to_tl(level_extent_px, &plane_tiling,
                            plane->nil.format,
                            plane->nil.sample_layout);

   /* Convert the extent and offset to tiles */
   const struct nil_Extent4D_Tiles bind_extent_tl =
      nil_extent4d_px_to_tl(bind_extent_px, &plane_tiling,
                            plane->nil.format,
                            plane->nil.sample_layout);
   const struct nil_Offset4D_Tiles bind_offset_tl =
      nil_offset4d_px_to_tl(bind_offset_px, &plane_tiling,
                            plane->nil.format,
                            plane->nil.sample_layout);

   image_bind_offset_B =
      nil_image_level_layer_offset_B(&plane->nil, level, layer);

   /* We can only bind contiguous ranges, so we'll split the image into rows
    * of tiles that are guaranteed to be contiguous, and bind in terms of
    * these rows
    */

   /* First, get the size of the bind. Since we have the extent in terms of
    * tiles already, we just need to multiply that by the tile size to get
    * the size in bytes
    */
   uint64_t row_bind_size_B = bind_extent_tl.width * tile_size_B;

   const uint32_t nvkmd_bind_count = bind_extent_tl.depth *
                                     bind_extent_tl.height;
   STACK_ARRAY(struct nvkmd_ctx_bind, nvkmd_binds, nvkmd_bind_count);
   uint32_t nvkmd_bind_idx = 0;

   /* Second, start walking the binding region in units of tiles, starting
    * from the third dimension
    */
   for (uint32_t z_tl = 0; z_tl < bind_extent_tl.depth; z_tl++) {
      /* Start walking the rows to be bound */
      for (uint32_t y_tl = 0; y_tl < bind_extent_tl.height; y_tl++) {
         /* For the bind offset, get a memory offset to the start of the row
          * in terms of the bind extent
          */
         const uint64_t mem_row_start_tl =
            y_tl * bind_extent_tl.width +
            z_tl * bind_extent_tl.width * bind_extent_tl.height;

         const uint32_t image_x_tl = bind_offset_tl.x;
         const uint32_t image_y_tl = bind_offset_tl.y + y_tl;
         const uint32_t image_z_tl = bind_offset_tl.z + z_tl;

         /* The image offset is calculated in terms of the level extent */
         const uint64_t image_row_start_tl =
            image_x_tl +
            image_y_tl * level_extent_tl.width +
            image_z_tl * level_extent_tl.width * level_extent_tl.height;

         nvkmd_binds[nvkmd_bind_idx++] = (struct nvkmd_ctx_bind) {
            .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
            .va = plane->va,
            .va_offset_B = image_bind_offset_B +
                           image_row_start_tl * tile_size_B,
            .mem = mem ? mem->mem : NULL,
            .mem_offset_B = mem_bind_offset_B +
                            mem_row_start_tl * tile_size_B,
            .range_B = row_bind_size_B,
         };
      }
   }

   assert(nvkmd_bind_idx == nvkmd_bind_count);
   VkResult result = nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base,
                                    nvkmd_bind_count, nvkmd_binds);

   STACK_ARRAY_FINISH(nvkmd_binds);

   return result;
}

VkResult
nvk_queue_image_bind(struct nvk_queue *queue,
                     const VkSparseImageMemoryBindInfo *bind_info)
{
   VK_FROM_HANDLE(nvk_image, image, bind_info->image);
   VkResult result;

   /* Sparse residency with multiplane is currently not supported */
   assert(image->plane_count == 1);

   for (unsigned i = 0; i < bind_info->bindCount; i++) {
      result = queue_image_plane_bind(queue, &image->planes[0],
                                      &bind_info->pBinds[i]);
      if (result != VK_SUCCESS)
         return result;
   }

   return VK_SUCCESS;
}

static bool
next_opaque_bind_plane(const VkSparseMemoryBind *bind,
                       uint64_t size_B, uint32_t align_B,
                       uint64_t *plane_offset_B,
                       uint64_t *mem_offset_B,
                       uint64_t *bind_size_B,
                       uint64_t *image_plane_offset_B_iter)
{
   /* Figure out the offset to thise plane and increment _iter up-front so
    * that we're free to early return elsewhere in the function.
    */
   *image_plane_offset_B_iter = align64(*image_plane_offset_B_iter, align_B);
   const uint64_t image_plane_offset_B = *image_plane_offset_B_iter;
   *image_plane_offset_B_iter += size_B;

   /* Offset into the image or image mip tail, as appropriate */
   uint64_t bind_offset_B = bind->resourceOffset;
   if (bind_offset_B >= NVK_MIP_TAIL_START_OFFSET)
      bind_offset_B -= NVK_MIP_TAIL_START_OFFSET;

   if (bind_offset_B < image_plane_offset_B) {
      /* The offset of the plane within the bind */
      const uint64_t bind_plane_offset_B =
         image_plane_offset_B - bind_offset_B;

      /* If this plane lies above the bound range, skip this plane */
      if (bind_plane_offset_B >= bind->size)
         return false;

      *plane_offset_B = 0;
      *mem_offset_B = bind->memoryOffset + bind_plane_offset_B;
      *bind_size_B = MIN2(bind->size - bind_plane_offset_B, size_B);
   } else {
      /* The offset of the bind within the plane */
      const uint64_t plane_bind_offset_B =
         bind_offset_B - image_plane_offset_B;

      /* If this plane lies below the bound range, skip this plane */
      if (plane_bind_offset_B >= size_B)
         return false;

      *plane_offset_B = plane_bind_offset_B;
      *mem_offset_B = bind->memoryOffset;
      *bind_size_B = MIN2(bind->size, size_B - plane_bind_offset_B);
   }

   return true;
}

static VkResult
queue_image_plane_opaque_bind(struct nvk_queue *queue,
                              struct nvk_image *image,
                              struct nvk_image_plane *plane,
                              const VkSparseMemoryBind *bind,
                              uint64_t *image_plane_offset_B)
{
   uint64_t plane_size_B, plane_align_B;
   nvk_image_plane_size_align_B(nvk_queue_device(queue), image, plane,
                                &plane_size_B, &plane_align_B);

   uint64_t plane_offset_B, mem_offset_B, bind_size_B;
   if (!next_opaque_bind_plane(bind, plane_size_B, plane_align_B,
                               &plane_offset_B, &mem_offset_B, &bind_size_B,
                               image_plane_offset_B))
      return VK_SUCCESS;

   VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);

   assert(plane_offset_B + bind_size_B <= plane->va->size_B);
   assert(!mem || mem_offset_B + bind_size_B <= mem->vk.size);

   const struct nvkmd_ctx_bind nvkmd_bind = {
      .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
      .va = plane->va,
      .va_offset_B = plane_offset_B,
      .mem = mem ? mem->mem : NULL,
      .mem_offset_B = mem_offset_B,
      .range_B = bind_size_B,
   };
   return nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base, 1, &nvkmd_bind);
}

static VkResult
queue_image_plane_bind_mip_tail(struct nvk_queue *queue,
                                struct nvk_image *image,
                                struct nvk_image_plane *plane,
                                const VkSparseMemoryBind *bind,
                                uint64_t *image_plane_offset_B)
{
   uint64_t plane_size_B, plane_align_B;
   nvk_image_plane_size_align_B(nvk_queue_device(queue), image, plane,
                                &plane_size_B, &plane_align_B);

   const uint64_t mip_tail_offset_B =
      nil_image_mip_tail_offset_B(&plane->nil);
   const uint64_t mip_tail_size_B =
      nil_image_mip_tail_size_B(&plane->nil);
   const uint64_t mip_tail_stride_B = plane->nil.array_stride_B;

   const uint64_t whole_mip_tail_size_B =
      mip_tail_size_B * plane->nil.extent_px.array_len;

   uint64_t plane_offset_B, mem_offset_B, bind_size_B;
   if (!next_opaque_bind_plane(bind, whole_mip_tail_size_B, plane_align_B,
                               &plane_offset_B, &mem_offset_B, &bind_size_B,
                               image_plane_offset_B))
      return VK_SUCCESS;

   VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);

   /* Range within the virtual mip_tail space */
   const uint64_t mip_bind_start_B = plane_offset_B;
   const uint64_t mip_bind_end_B = mip_bind_start_B + bind_size_B;

   /* Range of array slices covered by this bind */
   const uint32_t start_a = mip_bind_start_B / mip_tail_size_B;
   const uint32_t end_a = DIV_ROUND_UP(mip_bind_end_B, mip_tail_size_B);

   const uint32_t nvkmd_bind_count = end_a - start_a;
   STACK_ARRAY(struct nvkmd_ctx_bind, nvkmd_binds, nvkmd_bind_count);
   uint32_t nvkmd_bind_idx = 0;

   for (uint32_t a = start_a; a < end_a; a++) {
      /* Range within the virtual mip_tail space of this array slice */
      const uint64_t a_mip_bind_start_B =
         MAX2(a * mip_tail_size_B, mip_bind_start_B);
      const uint64_t a_mip_bind_end_B =
         MIN2((a + 1) * mip_tail_size_B, mip_bind_end_B);

      /* Offset and range within this mip_tail slice */
      const uint64_t a_offset_B = a_mip_bind_start_B - a * mip_tail_size_B;
      const uint64_t a_range_B = a_mip_bind_end_B - a_mip_bind_start_B;

      /* Offset within the current bind operation */
      const uint64_t a_bind_offset_B =
         a_mip_bind_start_B - mip_bind_start_B;

      /* Offset within the image */
      const uint64_t a_image_offset_B =
         mip_tail_offset_B + (a * mip_tail_stride_B) + a_offset_B;

      nvkmd_binds[nvkmd_bind_idx++] = (struct nvkmd_ctx_bind) {
         .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
         .va = plane->va,
         .va_offset_B = a_image_offset_B,
         .mem = mem ? mem->mem : NULL,
         .mem_offset_B = mem_offset_B + a_bind_offset_B,
         .range_B = a_range_B,
      };
   }

   assert(nvkmd_bind_idx == nvkmd_bind_count);
   VkResult result = nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base,
                                    nvkmd_bind_count, nvkmd_binds);

   STACK_ARRAY_FINISH(nvkmd_binds);

   return result;
}

VkResult
nvk_queue_image_opaque_bind(struct nvk_queue *queue,
                            const VkSparseImageOpaqueMemoryBindInfo *bind_info)
{
   VK_FROM_HANDLE(nvk_image, image, bind_info->image);
   VkResult result;

   for (unsigned i = 0; i < bind_info->bindCount; i++) {
      const VkSparseMemoryBind *bind = &bind_info->pBinds[i];

      uint64_t image_plane_offset_B = 0;
      for (unsigned plane = 0; plane < image->plane_count; plane++) {
         if (bind->resourceOffset >= NVK_MIP_TAIL_START_OFFSET) {
            result = queue_image_plane_bind_mip_tail(queue, image,
                                                     &image->planes[plane],
                                                     bind,
                                                     &image_plane_offset_B);
         } else {
            result = queue_image_plane_opaque_bind(queue, image,
                                                   &image->planes[plane],
                                                   bind,
                                                   &image_plane_offset_B);
         }
         if (result != VK_SUCCESS)
            return result;
      }
      if (image->stencil_copy_temp.nil.size_B > 0) {
         result = queue_image_plane_opaque_bind(queue, image,
                                                &image->stencil_copy_temp,
                                                bind,
                                                &image_plane_offset_B);
         if (result != VK_SUCCESS)
            return result;
      }
   }

   return VK_SUCCESS;
}

VKAPI_ATTR VkResult VKAPI_CALL
nvk_GetImageOpaqueCaptureDescriptorDataEXT(
    VkDevice _device,
    const VkImageCaptureDescriptorDataInfoEXT *pInfo,
    void *pData)
{
   return VK_SUCCESS;
}
