On Sat, Dec 9, 2017 at 6:08 AM, Felix Kuehling <Felix.Kuehling at amd.com> wrote:
> From: Harish Kasiviswanathan <Harish.Kasiviswanathan at amd.com>
>
> Generate and parse VCRAT tables for dGPUs in kfd_topology_add_device.
>
> Some information that isn't available in the CRAT table is patched
> into the topology after parsing.
>
> HSA_CAP_DOORBELL_TYPE_1_0 is dependent on the ASIC feature
> CP_HQD_PQ_CONTROL.SLOT_BASED_WPTR, which was not introduced in VI
> until Carrizo. Report HSA_CAP_DOORBELL_TYPE_PRE_1_0 on Tonga ASICs.
>
> Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan at amd.com>
> Signed-off-by: Ben Goz <ben.goz at amd.com>
> Signed-off-by: Amber Lin <Amber.Lin at amd.com>
> Signed-off-by: Jay Cornwall <Jay.Cornwall at amd.com>
> Signed-off-by: Kent Russell <kent.russell at amd.com>
> Signed-off-by: Felix Kuehling <Felix.Kuehling at amd.com>
> ---
> drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 594 +++++++++++++++++++++++++++++-
> drivers/gpu/drm/amd/amdkfd/kfd_crat.h | 5 +-
> drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 +
> drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 188 ++++++++--
> drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 8 +-
> 5 files changed, 746 insertions(+), 51 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> index 00732ec..ba7577b 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> @@ -20,10 +20,117 @@
> * OTHER DEALINGS IN THE SOFTWARE.
> */
> #include <linux/acpi.h>
> +#include <linux/amd-iommu.h>
> #include "kfd_crat.h"
> #include "kfd_priv.h"
> #include "kfd_topology.h"
>
> +/* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
> + * GPU processor ID are expressed with Bit[31]=1.
> + * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
> + * used in the CRAT.
> + */
> +static uint32_t gpu_processor_id_low = 0x80001000;
> +
> +/* Return the next available gpu_processor_id and increment it for next GPU
> + * @total_cu_count - Total CUs present in the GPU including ones
> + * masked off
> + */
> +static inline unsigned int get_and_inc_gpu_processor_id(
> + unsigned int total_cu_count)
> +{
> + int current_id = gpu_processor_id_low;
> +
> + gpu_processor_id_low += total_cu_count;
> + return current_id;
> +}
> +
> +/* Static table to describe GPU Cache information */
> +struct kfd_gpu_cache_info {
> + uint32_t cache_size;
> + uint32_t cache_level;
> + uint32_t flags;
> + /* Indicates how many Compute Units share this cache
> + * Value = 1 indicates the cache is not shared
> + */
> + uint32_t num_cu_shared;
> +};
> +
> +static struct kfd_gpu_cache_info kaveri_cache_info[] = {
> + {
> + /* TCP L1 Cache per CU */
> + .cache_size = 16,
> + .cache_level = 1,
> + .flags = (CRAT_CACHE_FLAGS_ENABLED |
> + CRAT_CACHE_FLAGS_DATA_CACHE |
> + CRAT_CACHE_FLAGS_SIMD_CACHE),
> + .num_cu_shared = 1,
> +
> + },
> + {
> + /* Scalar L1 Instruction Cache (in SQC module) per bank */
> + .cache_size = 16,
> + .cache_level = 1,
> + .flags = (CRAT_CACHE_FLAGS_ENABLED |
> + CRAT_CACHE_FLAGS_INST_CACHE |
> + CRAT_CACHE_FLAGS_SIMD_CACHE),
> + .num_cu_shared = 2,
> + },
> + {
> + /* Scalar L1 Data Cache (in SQC module) per bank */
> + .cache_size = 8,
> + .cache_level = 1,
> + .flags = (CRAT_CACHE_FLAGS_ENABLED |
> + CRAT_CACHE_FLAGS_DATA_CACHE |
> + CRAT_CACHE_FLAGS_SIMD_CACHE),
> + .num_cu_shared = 2,
> + },
> +
> + /* TODO: Add L2 Cache information */
> +};
> +
> +
> +static struct kfd_gpu_cache_info carrizo_cache_info[] = {
> + {
> + /* TCP L1 Cache per CU */
> + .cache_size = 16,
> + .cache_level = 1,
> + .flags = (CRAT_CACHE_FLAGS_ENABLED |
> + CRAT_CACHE_FLAGS_DATA_CACHE |
> + CRAT_CACHE_FLAGS_SIMD_CACHE),
> + .num_cu_shared = 1,
> + },
> + {
> + /* Scalar L1 Instruction Cache (in SQC module) per bank */
> + .cache_size = 8,
> + .cache_level = 1,
> + .flags = (CRAT_CACHE_FLAGS_ENABLED |
> + CRAT_CACHE_FLAGS_INST_CACHE |
> + CRAT_CACHE_FLAGS_SIMD_CACHE),
> + .num_cu_shared = 4,
> + },
> + {
> + /* Scalar L1 Data Cache (in SQC module) per bank. */
> + .cache_size = 4,
> + .cache_level = 1,
> + .flags = (CRAT_CACHE_FLAGS_ENABLED |
> + CRAT_CACHE_FLAGS_DATA_CACHE |
> + CRAT_CACHE_FLAGS_SIMD_CACHE),
> + .num_cu_shared = 4,
> + },
> +
> + /* TODO: Add L2 Cache information */
> +};
> +
> +/* NOTE: In future if more information is added to struct kfd_gpu_cache_info
> + * the following ASICs may need a separate table.
> + */
> +#define hawaii_cache_info kaveri_cache_info
> +#define tonga_cache_info carrizo_cache_info
> +#define fiji_cache_info carrizo_cache_info
> +#define polaris10_cache_info carrizo_cache_info
> +#define polaris11_cache_info carrizo_cache_info
> +
> static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
> struct crat_subtype_computeunit *cu)
> {
> @@ -44,7 +151,7 @@ static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
> dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
> dev->node_props.max_waves_per_simd = cu->max_waves_simd;
> dev->node_props.wave_front_size = cu->wave_front_size;
> - dev->node_props.array_count = cu->num_arrays;
> + dev->node_props.array_count = cu->array_count;
> dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
> dev->node_props.simd_per_cu = cu->num_simd_per_cu;
> dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
> @@ -94,9 +201,16 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
> if (!props)
> return -ENOMEM;
>
> - if (dev->node_props.cpu_cores_count == 0)
> - props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE;
> - else
> + /* We're on GPU node */
> + if (dev->node_props.cpu_cores_count == 0) {
> + /* APU */
> + if (mem->visibility_type == 0)
> + props->heap_type =
> + HSA_MEM_HEAP_TYPE_FB_PRIVATE;
> + /* dGPU */
> + else
> + props->heap_type = mem->visibility_type;
> + } else
> props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
>
> if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
> @@ -128,13 +242,29 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
> struct kfd_cache_properties *props;
> struct kfd_topology_device *dev;
> uint32_t id;
> + uint32_t total_num_of_cu;
>
> id = cache->processor_id_low;
>
> pr_debug("Found cache entry in CRAT table with processor_id=%d\n", id);
> - list_for_each_entry(dev, device_list, list)
> - if (id == dev->node_props.cpu_core_id_base ||
> - id == dev->node_props.simd_id_base) {
> + list_for_each_entry(dev, device_list, list) {
> + total_num_of_cu = (dev->node_props.array_count *
> + dev->node_props.cu_per_simd_array);
> +
> + /* Cache infomration in CRAT doesn't have proximity_domain
> + * information as it is associated with a CPU core or GPU
> + * Compute Unit. So map the cache using CPU core Id or SIMD
> + * (GPU) ID.
> + * TODO: This works because currently we can safely assume that
> + * Compute Units are parsed before caches are parsed. In
> + * future, remove this dependency
> + */
> + if ((id >= dev->node_props.cpu_core_id_base &&
> + id <= dev->node_props.cpu_core_id_base +
> + dev->node_props.cpu_cores_count) ||
> + (id >= dev->node_props.simd_id_base &&
> + id < dev->node_props.simd_id_base +
> + total_num_of_cu)) {
> props = kfd_alloc_struct(props);
> if (!props)
> return -ENOMEM;
> @@ -146,6 +276,8 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
> props->cachelines_per_tag = cache->lines_per_tag;
> props->cache_assoc = cache->associativity;
> props->cache_latency = cache->cache_latency;
> + memcpy(props->sibling_map, cache->sibling_map,
> + sizeof(props->sibling_map));
>
> if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
> props->cache_type |= HSA_CACHE_TYPE_DATA;
> @@ -162,6 +294,7 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
>
> break;
> }
> + }
>
> return 0;
> }
> @@ -172,8 +305,8 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
> static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
> struct list_head *device_list)
> {
> - struct kfd_iolink_properties *props;
> - struct kfd_topology_device *dev;
> + struct kfd_iolink_properties *props = NULL, *props2;
> + struct kfd_topology_device *dev, *cpu_dev;
> uint32_t id_from;
> uint32_t id_to;
>
> @@ -192,11 +325,12 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
> props->node_to = id_to;
> props->ver_maj = iolink->version_major;
> props->ver_min = iolink->version_minor;
> + props->iolink_type = iolink->io_interface_type;
>
> - /*
> - * weight factor (derived from CDIR), currently always 1
> - */
> - props->weight = 1;
> + if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
> + props->weight = 20;
> + else
> + props->weight = node_distance(id_from, id_to);
>
> props->min_latency = iolink->minimum_latency;
> props->max_latency = iolink->maximum_latency;
> @@ -208,11 +342,29 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
> dev->io_link_count++;
> dev->node_props.io_links_count++;
> list_add_tail(&props->list, &dev->io_link_props);
> -
> break;
> }
> }
>
> + /* CPU topology is created before GPUs are detected, so CPU->GPU
> + * links are not built at that time. If a PCIe type is discovered, it
> + * means a GPU is detected and we are adding GPU->CPU to the topology.
> + * At this time, also add the corresponded CPU->GPU link.
> + */
> + if (props && props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) {
> + cpu_dev = kfd_topology_device_by_proximity_domain(id_to);
> + if (!cpu_dev)
> + return -ENODEV;
> + /* same everything but the other direction */
> + props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
> + props2->node_from = id_to;
> + props2->node_to = id_from;
> + props2->kobj = NULL;
> + cpu_dev->io_link_count++;
> + cpu_dev->node_props.io_links_count++;
> + list_add_tail(&props2->list, &cpu_dev->io_link_props);
> + }
> +
> return 0;
> }
>
> @@ -338,6 +490,176 @@ int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
> return ret;
> }
>
> +/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
> +static int fill_in_pcache(struct crat_subtype_cache *pcache,
> + struct kfd_gpu_cache_info *pcache_info,
> + struct kfd_cu_info *cu_info,
> + int mem_available,
> + int cu_bitmask,
> + int cache_type, unsigned int cu_processor_id,
> + int cu_block)
> +{
> + unsigned int cu_sibling_map_mask;
> + int first_active_cu;
> +
> + /* First check if enough memory is available */
> + if (sizeof(struct crat_subtype_cache) > mem_available)
> + return -ENOMEM;
> +
> + cu_sibling_map_mask = cu_bitmask;
> + cu_sibling_map_mask >>= cu_block;
> + cu_sibling_map_mask &=
> + ((1 << pcache_info[cache_type].num_cu_shared) - 1);
> + first_active_cu = ffs(cu_sibling_map_mask);
> +
> + /* CU could be inactive. In case of shared cache find the first active
> + * CU. and incase of non-shared cache check if the CU is inactive. If
> + * inactive active skip it
> + */
> + if (first_active_cu) {
> + memset(pcache, 0, sizeof(struct crat_subtype_cache));
> + pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
> + pcache->length = sizeof(struct crat_subtype_cache);
> + pcache->flags = pcache_info[cache_type].flags;
> + pcache->processor_id_low = cu_processor_id
> + + (first_active_cu - 1);
> + pcache->cache_level = pcache_info[cache_type].cache_level;
> + pcache->cache_size = pcache_info[cache_type].cache_size;
> +
> + /* Sibling map is w.r.t processor_id_low, so shift out
> + * inactive CU
> + */
> + cu_sibling_map_mask =
> + cu_sibling_map_mask >> (first_active_cu - 1);
> +
> + pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
> + pcache->sibling_map[1] =
> + (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
> + pcache->sibling_map[2] =
> + (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
> + pcache->sibling_map[3] =
> + (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
> + return 0;
> + }
> + return 1;
> +}
> +
> +/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
> + * tables
> + *
> + * @kdev - [IN] GPU device
> + * @gpu_processor_id - [IN] GPU processor ID to which these caches
> + * associate
> + * @available_size - [IN] Amount of memory available in pcache
> + * @cu_info - [IN] Compute Unit info obtained from KGD
> + * @pcache - [OUT] memory into which cache data is to be filled in.
> + * @size_filled - [OUT] amount of data used up in pcache.
> + * @num_of_entries - [OUT] number of caches added
> + */
> +static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
> + int gpu_processor_id,
> + int available_size,
> + struct kfd_cu_info *cu_info,
> + struct crat_subtype_cache *pcache,
> + int *size_filled,
> + int *num_of_entries)
> +{
> + struct kfd_gpu_cache_info *pcache_info;
> + int num_of_cache_types = 0;
> + int i, j, k;
> + int ct = 0;
> + int mem_available = available_size;
> + unsigned int cu_processor_id;
> + int ret;
> +
> + switch (kdev->device_info->asic_family) {
> + case CHIP_KAVERI:
> + pcache_info = kaveri_cache_info;
> + num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
> + break;
> + case CHIP_HAWAII:
> + pcache_info = hawaii_cache_info;
> + num_of_cache_types = ARRAY_SIZE(hawaii_cache_info);
> + break;
> + case CHIP_CARRIZO:
> + pcache_info = carrizo_cache_info;
> + num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
> + break;
> + case CHIP_TONGA:
> + pcache_info = tonga_cache_info;
> + num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
> + break;
> + case CHIP_FIJI:
> + pcache_info = fiji_cache_info;
> + num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
> + break;
> + case CHIP_POLARIS10:
> + pcache_info = polaris10_cache_info;
> + num_of_cache_types = ARRAY_SIZE(polaris10_cache_info);
> + break;
> + case CHIP_POLARIS11:
> + pcache_info = polaris11_cache_info;
> + num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
> + break;
> + default:
> + return -EINVAL;
> + }
> +
> + *size_filled = 0;
> + *num_of_entries = 0;
> +
> + /* For each type of cache listed in the kfd_gpu_cache_info table,
> + * go through all available Compute Units.
> + * The [i,j,k] loop will
> + * if kfd_gpu_cache_info.num_cu_shared = 1
> + * will parse through all available CU
> + * If (kfd_gpu_cache_info.num_cu_shared != 1)
> + * then it will consider only one CU from
> + * the shared unit
> + */
> +
> + for (ct = 0; ct < num_of_cache_types; ct++) {
> + cu_processor_id = gpu_processor_id;
> + for (i = 0; i < cu_info->num_shader_engines; i++) {
> + for (j = 0; j < cu_info->num_shader_arrays_per_engine;
> + j++) {
> + for (k = 0; k < cu_info->num_cu_per_sh;
> + k += pcache_info[ct].num_cu_shared) {
> +
> + ret = fill_in_pcache(pcache,
> + pcache_info,
> + cu_info,
> + mem_available,
> + cu_info->cu_bitmap[i][j],
> + ct,
> + cu_processor_id,
> + k);
> +
> + if (ret < 0)
> + break;
> +
> + if (!ret) {
> + pcache++;
> + (*num_of_entries)++;
> + mem_available -=
> + sizeof(*pcache);
> + (*size_filled) +=
> + sizeof(*pcache);
> + }
> +
> + /* Move to next CU block */
> + cu_processor_id +=
> + pcache_info[ct].num_cu_shared;
> + }
> + }
> + }
> + }
> +
> + pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
> +
> + return 0;
> +}
> +
> /*
> * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
> * copies CRAT from ACPI (if available).
> @@ -624,6 +946,239 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
> return 0;
> }
>
> +static int kfd_fill_gpu_memory_affinity(int *avail_size,
> + struct kfd_dev *kdev, uint8_t type, uint64_t size,
> + struct crat_subtype_memory *sub_type_hdr,
> + uint32_t proximity_domain,
> + const struct kfd_local_mem_info *local_mem_info)
> +{
> + *avail_size -= sizeof(struct crat_subtype_memory);
> + if (*avail_size < 0)
> + return -ENOMEM;
> +
> + memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
> + sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
> + sub_type_hdr->length = sizeof(struct crat_subtype_memory);
> + sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
> +
> + sub_type_hdr->proximity_domain = proximity_domain;
> +
> + pr_debug("Fill gpu memory affinity - type 0x%x size 0x%llx\n",
> + type, size);
> +
> + sub_type_hdr->length_low = lower_32_bits(size);
> + sub_type_hdr->length_high = upper_32_bits(size);
> +
> + sub_type_hdr->width = local_mem_info->vram_width;
> + sub_type_hdr->visibility_type = type;
> +
> + return 0;
> +}
> +
> +/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
> + * to its NUMA node
> + * @avail_size: Available size in the memory
> + * @kdev - [IN] GPU device
> + * @sub_type_hdr: Memory into which io link info will be filled in
> + * @proximity_domain - proximity domain of the GPU node
> + *
> + * Return 0 if successful else return -ve value
> + */
> +static int kfd_fill_gpu_direct_io_link(int *avail_size,
> + struct kfd_dev *kdev,
> + struct crat_subtype_iolink *sub_type_hdr,
> + uint32_t proximity_domain)
> +{
> + *avail_size -= sizeof(struct crat_subtype_iolink);
> + if (*avail_size < 0)
> + return -ENOMEM;
> +
> + memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
> +
> + /* Fill in subtype header data */
> + sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
> + sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
> + sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
> +
> + /* Fill in IOLINK subtype.
> + * TODO: Fill-in other fields of iolink subtype
> + */
> + sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
> + sub_type_hdr->proximity_domain_from = proximity_domain;
> +#ifdef CONFIG_NUMA
> + if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
Had to add #include <linux/pci.h> at the head of the file to make this
line compile
Oded
> + sub_type_hdr->proximity_domain_to = 0;
> + else
> + sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node;
> +#else
> + sub_type_hdr->proximity_domain_to = 0;
> +#endif
> + return 0;
> +}
> +
> +/* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
> + *
> + * @pcrat_image: Fill in VCRAT for GPU
> + * @size: [IN] allocated size of crat_image.
> + * [OUT] actual size of data filled in crat_image
> + */
> +static int kfd_create_vcrat_image_gpu(void *pcrat_image,
> + size_t *size, struct kfd_dev *kdev,
> + uint32_t proximity_domain)
> +{
> + struct crat_header *crat_table = (struct crat_header *)pcrat_image;
> + struct crat_subtype_generic *sub_type_hdr;
> + struct crat_subtype_computeunit *cu;
> + struct kfd_cu_info cu_info;
> + struct amd_iommu_device_info iommu_info;
> + int avail_size = *size;
> + uint32_t total_num_of_cu;
> + int num_of_cache_entries = 0;
> + int cache_mem_filled = 0;
> + int ret = 0;
> + const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
> + AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
> + AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
> + struct kfd_local_mem_info local_mem_info;
> +
> + if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
> + return -EINVAL;
> +
> + /* Fill the CRAT Header.
> + * Modify length and total_entries as subunits are added.
> + */
> + avail_size -= sizeof(struct crat_header);
> + if (avail_size < 0)
> + return -ENOMEM;
> +
> + memset(crat_table, 0, sizeof(struct crat_header));
> +
> + memcpy(&crat_table->signature, CRAT_SIGNATURE,
> + sizeof(crat_table->signature));
> + /* Change length as we add more subtypes*/
> + crat_table->length = sizeof(struct crat_header);
> + crat_table->num_domains = 1;
> + crat_table->total_entries = 0;
> +
> + /* Fill in Subtype: Compute Unit
> + * First fill in the sub type header and then sub type data
> + */
> + avail_size -= sizeof(struct crat_subtype_computeunit);
> + if (avail_size < 0)
> + return -ENOMEM;
> +
> + sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
> + memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
> +
> + sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
> + sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
> + sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
> +
> + /* Fill CU subtype data */
> + cu = (struct crat_subtype_computeunit *)sub_type_hdr;
> + cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
> + cu->proximity_domain = proximity_domain;
> +
> + kdev->kfd2kgd->get_cu_info(kdev->kgd, &cu_info);
> + cu->num_simd_per_cu = cu_info.simd_per_cu;
> + cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
> + cu->max_waves_simd = cu_info.max_waves_per_simd;
> +
> + cu->wave_front_size = cu_info.wave_front_size;
> + cu->array_count = cu_info.num_shader_arrays_per_engine *
> + cu_info.num_shader_engines;
> + total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
> + cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
> + cu->num_cu_per_array = cu_info.num_cu_per_sh;
> + cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
> + cu->num_banks = cu_info.num_shader_engines;
> + cu->lds_size_in_kb = cu_info.lds_size;
> +
> + cu->hsa_capability = 0;
> +
> + /* Check if this node supports IOMMU. During parsing this flag will
> + * translate to HSA_CAP_ATS_PRESENT
> + */
> + iommu_info.flags = 0;
> + if (amd_iommu_device_info(kdev->pdev, &iommu_info) == 0) {
> + if ((iommu_info.flags & required_iommu_flags) ==
> + required_iommu_flags)
> + cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
> + }
> +
> + crat_table->length += sub_type_hdr->length;
> + crat_table->total_entries++;
> +
> + /* Fill in Subtype: Memory. Only on systems with large BAR (no
> + * private FB), report memory as public. On other systems
> + * report the total FB size (public+private) as a single
> + * private heap.
> + */
> + kdev->kfd2kgd->get_local_mem_info(kdev->kgd, &local_mem_info);
> + sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
> + sub_type_hdr->length);
> +
> + if (local_mem_info.local_mem_size_private == 0)
> + ret = kfd_fill_gpu_memory_affinity(&avail_size,
> + kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
> + local_mem_info.local_mem_size_public,
> + (struct crat_subtype_memory *)sub_type_hdr,
> + proximity_domain,
> + &local_mem_info);
> + else
> + ret = kfd_fill_gpu_memory_affinity(&avail_size,
> + kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
> + local_mem_info.local_mem_size_public +
> + local_mem_info.local_mem_size_private,
> + (struct crat_subtype_memory *)sub_type_hdr,
> + proximity_domain,
> + &local_mem_info);
> + if (ret < 0)
> + return ret;
> +
> + crat_table->length += sizeof(struct crat_subtype_memory);
> + crat_table->total_entries++;
> +
> + /* TODO: Fill in cache information. This information is NOT readily
> + * available in KGD
> + */
> + sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
> + sub_type_hdr->length);
> + ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
> + avail_size,
> + &cu_info,
> + (struct crat_subtype_cache *)sub_type_hdr,
> + &cache_mem_filled,
> + &num_of_cache_entries);
> +
> + if (ret < 0)
> + return ret;
> +
> + crat_table->length += cache_mem_filled;
> + crat_table->total_entries += num_of_cache_entries;
> + avail_size -= cache_mem_filled;
> +
> + /* Fill in Subtype: IO_LINKS
> + * Only direct links are added here which is Link from GPU to
> + * to its NUMA node. Indirect links are added by userspace.
> + */
> + sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
> + cache_mem_filled);
> + ret = kfd_fill_gpu_direct_io_link(&avail_size, kdev,
> + (struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
> +
> + if (ret < 0)
> + return ret;
> +
> + crat_table->length += sub_type_hdr->length;
> + crat_table->total_entries++;
> +
> + *size = crat_table->length;
> + pr_info("Virtual CRAT table created for GPU\n");
> +
> + return ret;
> +}
> +
> /* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
> * creates a Virtual CRAT (VCRAT) image
> *
> @@ -667,9 +1222,14 @@ int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
> ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
> break;
> case COMPUTE_UNIT_GPU:
> - /* TODO: */
> - ret = -EINVAL;
> - pr_err("VCRAT not implemented for dGPU\n");
> + if (!kdev)
> + return -EINVAL;
> + pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
> + if (!pcrat_image)
> + return -ENOMEM;
> + *size = VCRAT_SIZE_FOR_GPU;
> + ret = kfd_create_vcrat_image_gpu(pcrat_image, size, kdev,
> + proximity_domain);
> break;
> case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU):
> /* TODO: */
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
> index aaa43ab..c97979c 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
> @@ -109,7 +109,7 @@ struct crat_subtype_computeunit {
> uint8_t wave_front_size;
> uint8_t num_banks;
> uint16_t micro_engine_id;
> - uint8_t num_arrays;
> + uint8_t array_count;
> uint8_t num_cu_per_array;
> uint8_t num_simd_per_cu;
> uint8_t max_slots_scatch_cu;
> @@ -137,7 +137,8 @@ struct crat_subtype_memory {
> uint32_t length_low;
> uint32_t length_high;
> uint32_t width;
> - uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH];
> + uint8_t visibility_type; /* for virtual (dGPU) CRAT */
> + uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH - 1];
> };
>
> /*
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index aeee9d4..f0327c2 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -668,6 +668,8 @@ int kfd_topology_init(void);
> void kfd_topology_shutdown(void);
> int kfd_topology_add_device(struct kfd_dev *gpu);
> int kfd_topology_remove_device(struct kfd_dev *gpu);
> +struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
> + uint32_t proximity_domain);
> struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
> struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
> int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> index 9aa6004..7fe7ee0 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> @@ -43,6 +43,25 @@ static struct kfd_system_properties sys_props;
> static DECLARE_RWSEM(topology_lock);
> static atomic_t topology_crat_proximity_domain;
>
> +struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
> + uint32_t proximity_domain)
> +{
> + struct kfd_topology_device *top_dev;
> + struct kfd_topology_device *device = NULL;
> +
> + down_read(&topology_lock);
> +
> + list_for_each_entry(top_dev, &topology_device_list, list)
> + if (top_dev->proximity_domain == proximity_domain) {
> + device = top_dev;
> + break;
> + }
> +
> + up_read(&topology_lock);
> +
> + return device;
> +}
> +
> struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
> {
> struct kfd_topology_device *top_dev;
> @@ -79,6 +98,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
> return device;
> }
>
> +/* Called with write topology_lock acquired */
> static void kfd_release_topology_device(struct kfd_topology_device *dev)
> {
> struct kfd_mem_properties *mem;
> @@ -394,8 +414,7 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
> }
>
> sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
> - dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(
> - dev->gpu->kgd));
> + dev->node_props.max_engine_clk_fcompute);
>
> sysfs_show_64bit_prop(buffer, "local_mem_size",
> (unsigned long long int) 0);
> @@ -597,6 +616,7 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
> return 0;
> }
>
> +/* Called with write topology lock acquired */
> static int kfd_build_sysfs_node_tree(void)
> {
> struct kfd_topology_device *dev;
> @@ -613,6 +633,7 @@ static int kfd_build_sysfs_node_tree(void)
> return 0;
> }
>
> +/* Called with write topology lock acquired */
> static void kfd_remove_sysfs_node_tree(void)
> {
> struct kfd_topology_device *dev;
> @@ -908,19 +929,26 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
>
> return hashout;
> }
> -
> +/* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
> + * the GPU device is not already present in the topology device
> + * list then return NULL. This means a new topology device has to
> + * be created for this GPU.
> + * TODO: Rather than assiging @gpu to first topology device withtout
> + * gpu attached, it will better to have more stringent check.
> + */
> static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
> {
> struct kfd_topology_device *dev;
> struct kfd_topology_device *out_dev = NULL;
>
> + down_write(&topology_lock);
> list_for_each_entry(dev, &topology_device_list, list)
> if (!dev->gpu && (dev->node_props.simd_count > 0)) {
> dev->gpu = gpu;
> out_dev = dev;
> break;
> }
> -
> + up_write(&topology_lock);
> return out_dev;
> }
>
> @@ -932,6 +960,45 @@ static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
> */
> }
>
> +/* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
> + * patch this after CRAT parsing.
> + */
> +static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
> +{
> + struct kfd_mem_properties *mem;
> + struct kfd_local_mem_info local_mem_info;
> +
> + if (!dev)
> + return;
> +
> + /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
> + * single bank of VRAM local memory.
> + * for dGPUs - VCRAT reports only one bank of Local Memory
> + * for APUs - If CRAT from ACPI reports more than one bank, then
> + * all the banks will report the same mem_clk_max information
> + */
> + dev->gpu->kfd2kgd->get_local_mem_info(dev->gpu->kgd,
> + &local_mem_info);
> +
> + list_for_each_entry(mem, &dev->mem_props, list)
> + mem->mem_clk_max = local_mem_info.mem_clk_max;
> +}
> +
> +static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
> +{
> + struct kfd_iolink_properties *link;
> +
> + if (!dev || !dev->gpu)
> + return;
> +
> + /* GPU only creates direck links so apply flags setting to all */
> + if (dev->gpu->device_info->asic_family == CHIP_HAWAII)
> + list_for_each_entry(link, &dev->io_link_props, list)
> + link->flags = CRAT_IOLINK_FLAGS_ENABLED |
> + CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
> + CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
> +}
> +
> int kfd_topology_add_device(struct kfd_dev *gpu)
> {
> uint32_t gpu_id;
> @@ -939,6 +1006,9 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
> struct kfd_cu_info cu_info;
> int res = 0;
> struct list_head temp_topology_device_list;
> + void *crat_image = NULL;
> + size_t image_size = 0;
> + int proximity_domain;
>
> INIT_LIST_HEAD(&temp_topology_device_list);
>
> @@ -946,27 +1016,33 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
>
> pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
>
> - /*
> - * Try to assign the GPU to existing topology device (generated from
> - * CRAT table
> + proximity_domain = atomic_inc_return(&topology_crat_proximity_domain);
> +
> + /* Check to see if this gpu device exists in the topology_device_list.
> + * If so, assign the gpu to that device,
> + * else create a Virtual CRAT for this gpu device and then parse that
> + * CRAT to create a new topology device. Once created assign the gpu to
> + * that topology device
> */
> dev = kfd_assign_gpu(gpu);
> if (!dev) {
> - pr_info("GPU was not found in the current topology. Extending.\n");
> - kfd_debug_print_topology();
> - dev = kfd_create_topology_device(&temp_topology_device_list);
> - if (!dev) {
> - res = -ENOMEM;
> + res = kfd_create_crat_image_virtual(&crat_image, &image_size,
> + COMPUTE_UNIT_GPU, gpu,
> + proximity_domain);
> + if (res) {
> + pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
> + gpu_id);
> + return res;
> + }
> + res = kfd_parse_crat_table(crat_image,
> + &temp_topology_device_list,
> + proximity_domain);
> + if (res) {
> + pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
> + gpu_id);
> goto err;
> }
>
> - dev->gpu = gpu;
> -
> - /*
> - * TODO: Make a call to retrieve topology information from the
> - * GPU vBIOS
> - */
> -
> down_write(&topology_lock);
> kfd_topology_update_device_list(&temp_topology_device_list,
> &topology_device_list);
> @@ -974,34 +1050,86 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
> /* Update the SYSFS tree, since we added another topology
> * device
> */
> - if (kfd_topology_update_sysfs() < 0)
> - kfd_topology_release_sysfs();
> -
> + res = kfd_topology_update_sysfs();
> up_write(&topology_lock);
>
> + if (!res)
> + sys_props.generation_count++;
> + else
> + pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
> + gpu_id, res);
> + dev = kfd_assign_gpu(gpu);
> + if (WARN_ON(!dev)) {
> + res = -ENODEV;
> + goto err;
> + }
> }
>
> dev->gpu_id = gpu_id;
> gpu->id = gpu_id;
> +
> + /* TODO: Move the following lines to function
> + * kfd_add_non_crat_information
> + */
> +
> + /* Fill-in additional information that is not available in CRAT but
> + * needed for the topology
> + */
> +
> dev->gpu->kfd2kgd->get_cu_info(dev->gpu->kgd, &cu_info);
> - dev->node_props.simd_count = dev->node_props.simd_per_cu *
> - cu_info.cu_active_number;
> + dev->node_props.simd_arrays_per_engine =
> + cu_info.num_shader_arrays_per_engine;
> +
> dev->node_props.vendor_id = gpu->pdev->vendor;
> dev->node_props.device_id = gpu->pdev->device;
> dev->node_props.location_id = PCI_DEVID(gpu->pdev->bus->number,
> gpu->pdev->devfn);
> - /*
> - * TODO: Retrieve max engine clock values from KGD
> - */
> -
> - if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
> - dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE;
> + dev->node_props.max_engine_clk_fcompute =
> + dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(dev->gpu->kgd);
> + dev->node_props.max_engine_clk_ccompute =
> + cpufreq_quick_get_max(0) / 1000;
> +
> + kfd_fill_mem_clk_max_info(dev);
> + kfd_fill_iolink_non_crat_info(dev);
> +
> + switch (dev->gpu->device_info->asic_family) {
> + case CHIP_KAVERI:
> + case CHIP_HAWAII:
> + case CHIP_TONGA:
> + dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
> + HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
> + HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
> + break;
> + case CHIP_CARRIZO:
> + case CHIP_FIJI:
> + case CHIP_POLARIS10:
> + case CHIP_POLARIS11:
> pr_debug("Adding doorbell packet type capability\n");
> + dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
> + HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
> + HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
> + break;
> + default:
> + WARN(1, "Unexpected ASIC family %u",
> + dev->gpu->device_info->asic_family);
> }
>
> + /* Fix errors in CZ CRAT.
> + * simd_count: Carrizo CRAT reports wrong simd_count, probably
> + * because it doesn't consider masked out CUs
> + * capability flag: Carrizo CRAT doesn't report IOMMU
> + * flags. TODO: Fix this.
> + */
> + if (dev->gpu->device_info->asic_family == CHIP_CARRIZO)
> + dev->node_props.simd_count =
> + cu_info.simd_per_cu * cu_info.cu_active_number;
> +
> + kfd_debug_print_topology();
> +
> if (!res)
> kfd_notify_gpu_change(gpu_id, 1);
> err:
> + kfd_destroy_crat_image(crat_image);
> return res;
> }
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
> index 8668189..55de56f 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
> @@ -39,8 +39,12 @@
> #define HSA_CAP_WATCH_POINTS_SUPPORTED 0x00000080
> #define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00
> #define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8
> -#define HSA_CAP_RESERVED 0xfffff000
> -#define HSA_CAP_DOORBELL_PACKET_TYPE 0x00001000
> +#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK 0x00003000
> +#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT 12
> +#define HSA_CAP_RESERVED 0xffffc000
> +
> +#define HSA_CAP_DOORBELL_TYPE_PRE_1_0 0x0
> +#define HSA_CAP_DOORBELL_TYPE_1_0 0x1
>
> struct kfd_node_properties {
> uint32_t cpu_cores_count;
> --
> 2.7.4
>
