1 /* $NetBSD: kfd_crat.c,v 1.2 2021/12/18 23:44:59 riastradh Exp $ */
2
3 /*
4 * Copyright 2015-2017 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(0, "$NetBSD: kfd_crat.c,v 1.2 2021/12/18 23:44:59 riastradh Exp $");
27
28 #include <linux/pci.h>
29 #include <linux/acpi.h>
30 #include "kfd_crat.h"
31 #include "kfd_priv.h"
32 #include "kfd_topology.h"
33 #include "kfd_iommu.h"
34 #include "amdgpu_amdkfd.h"
35
36 /* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
37 * GPU processor ID are expressed with Bit[31]=1.
38 * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
39 * used in the CRAT.
40 */
41 static uint32_t gpu_processor_id_low = 0x80001000;
42
43 /* Return the next available gpu_processor_id and increment it for next GPU
44 * @total_cu_count - Total CUs present in the GPU including ones
45 * masked off
46 */
get_and_inc_gpu_processor_id(unsigned int total_cu_count)47 static inline unsigned int get_and_inc_gpu_processor_id(
48 unsigned int total_cu_count)
49 {
50 int current_id = gpu_processor_id_low;
51
52 gpu_processor_id_low += total_cu_count;
53 return current_id;
54 }
55
56 /* Static table to describe GPU Cache information */
57 struct kfd_gpu_cache_info {
58 uint32_t cache_size;
59 uint32_t cache_level;
60 uint32_t flags;
61 /* Indicates how many Compute Units share this cache
62 * Value = 1 indicates the cache is not shared
63 */
64 uint32_t num_cu_shared;
65 };
66
67 static struct kfd_gpu_cache_info kaveri_cache_info[] = {
68 {
69 /* TCP L1 Cache per CU */
70 .cache_size = 16,
71 .cache_level = 1,
72 .flags = (CRAT_CACHE_FLAGS_ENABLED |
73 CRAT_CACHE_FLAGS_DATA_CACHE |
74 CRAT_CACHE_FLAGS_SIMD_CACHE),
75 .num_cu_shared = 1,
76
77 },
78 {
79 /* Scalar L1 Instruction Cache (in SQC module) per bank */
80 .cache_size = 16,
81 .cache_level = 1,
82 .flags = (CRAT_CACHE_FLAGS_ENABLED |
83 CRAT_CACHE_FLAGS_INST_CACHE |
84 CRAT_CACHE_FLAGS_SIMD_CACHE),
85 .num_cu_shared = 2,
86 },
87 {
88 /* Scalar L1 Data Cache (in SQC module) per bank */
89 .cache_size = 8,
90 .cache_level = 1,
91 .flags = (CRAT_CACHE_FLAGS_ENABLED |
92 CRAT_CACHE_FLAGS_DATA_CACHE |
93 CRAT_CACHE_FLAGS_SIMD_CACHE),
94 .num_cu_shared = 2,
95 },
96
97 /* TODO: Add L2 Cache information */
98 };
99
100
101 static struct kfd_gpu_cache_info carrizo_cache_info[] = {
102 {
103 /* TCP L1 Cache per CU */
104 .cache_size = 16,
105 .cache_level = 1,
106 .flags = (CRAT_CACHE_FLAGS_ENABLED |
107 CRAT_CACHE_FLAGS_DATA_CACHE |
108 CRAT_CACHE_FLAGS_SIMD_CACHE),
109 .num_cu_shared = 1,
110 },
111 {
112 /* Scalar L1 Instruction Cache (in SQC module) per bank */
113 .cache_size = 8,
114 .cache_level = 1,
115 .flags = (CRAT_CACHE_FLAGS_ENABLED |
116 CRAT_CACHE_FLAGS_INST_CACHE |
117 CRAT_CACHE_FLAGS_SIMD_CACHE),
118 .num_cu_shared = 4,
119 },
120 {
121 /* Scalar L1 Data Cache (in SQC module) per bank. */
122 .cache_size = 4,
123 .cache_level = 1,
124 .flags = (CRAT_CACHE_FLAGS_ENABLED |
125 CRAT_CACHE_FLAGS_DATA_CACHE |
126 CRAT_CACHE_FLAGS_SIMD_CACHE),
127 .num_cu_shared = 4,
128 },
129
130 /* TODO: Add L2 Cache information */
131 };
132
133 /* NOTE: In future if more information is added to struct kfd_gpu_cache_info
134 * the following ASICs may need a separate table.
135 */
136 #define hawaii_cache_info kaveri_cache_info
137 #define tonga_cache_info carrizo_cache_info
138 #define fiji_cache_info carrizo_cache_info
139 #define polaris10_cache_info carrizo_cache_info
140 #define polaris11_cache_info carrizo_cache_info
141 #define polaris12_cache_info carrizo_cache_info
142 #define vegam_cache_info carrizo_cache_info
143 /* TODO - check & update Vega10 cache details */
144 #define vega10_cache_info carrizo_cache_info
145 #define raven_cache_info carrizo_cache_info
146 #define renoir_cache_info carrizo_cache_info
147 /* TODO - check & update Navi10 cache details */
148 #define navi10_cache_info carrizo_cache_info
149
kfd_populated_cu_info_cpu(struct kfd_topology_device * dev,struct crat_subtype_computeunit * cu)150 static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
151 struct crat_subtype_computeunit *cu)
152 {
153 dev->node_props.cpu_cores_count = cu->num_cpu_cores;
154 dev->node_props.cpu_core_id_base = cu->processor_id_low;
155 if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
156 dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
157
158 pr_debug("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
159 cu->processor_id_low);
160 }
161
kfd_populated_cu_info_gpu(struct kfd_topology_device * dev,struct crat_subtype_computeunit * cu)162 static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
163 struct crat_subtype_computeunit *cu)
164 {
165 dev->node_props.simd_id_base = cu->processor_id_low;
166 dev->node_props.simd_count = cu->num_simd_cores;
167 dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
168 dev->node_props.max_waves_per_simd = cu->max_waves_simd;
169 dev->node_props.wave_front_size = cu->wave_front_size;
170 dev->node_props.array_count = cu->array_count;
171 dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
172 dev->node_props.simd_per_cu = cu->num_simd_per_cu;
173 dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
174 if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
175 dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
176 pr_debug("CU GPU: id_base=%d\n", cu->processor_id_low);
177 }
178
179 /* kfd_parse_subtype_cu - parse compute unit subtypes and attach it to correct
180 * topology device present in the device_list
181 */
kfd_parse_subtype_cu(struct crat_subtype_computeunit * cu,struct list_head * device_list)182 static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
183 struct list_head *device_list)
184 {
185 struct kfd_topology_device *dev;
186
187 pr_debug("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
188 cu->proximity_domain, cu->hsa_capability);
189 list_for_each_entry(dev, device_list, list) {
190 if (cu->proximity_domain == dev->proximity_domain) {
191 if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
192 kfd_populated_cu_info_cpu(dev, cu);
193
194 if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
195 kfd_populated_cu_info_gpu(dev, cu);
196 break;
197 }
198 }
199
200 return 0;
201 }
202
203 static struct kfd_mem_properties *
find_subtype_mem(uint32_t heap_type,uint32_t flags,uint32_t width,struct kfd_topology_device * dev)204 find_subtype_mem(uint32_t heap_type, uint32_t flags, uint32_t width,
205 struct kfd_topology_device *dev)
206 {
207 struct kfd_mem_properties *props;
208
209 list_for_each_entry(props, &dev->mem_props, list) {
210 if (props->heap_type == heap_type
211 && props->flags == flags
212 && props->width == width)
213 return props;
214 }
215
216 return NULL;
217 }
218 /* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
219 * topology device present in the device_list
220 */
kfd_parse_subtype_mem(struct crat_subtype_memory * mem,struct list_head * device_list)221 static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
222 struct list_head *device_list)
223 {
224 struct kfd_mem_properties *props;
225 struct kfd_topology_device *dev;
226 uint32_t heap_type;
227 uint64_t size_in_bytes;
228 uint32_t flags = 0;
229 uint32_t width;
230
231 pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
232 mem->proximity_domain);
233 list_for_each_entry(dev, device_list, list) {
234 if (mem->proximity_domain == dev->proximity_domain) {
235 /* We're on GPU node */
236 if (dev->node_props.cpu_cores_count == 0) {
237 /* APU */
238 if (mem->visibility_type == 0)
239 heap_type =
240 HSA_MEM_HEAP_TYPE_FB_PRIVATE;
241 /* dGPU */
242 else
243 heap_type = mem->visibility_type;
244 } else
245 heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
246
247 if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
248 flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
249 if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
250 flags |= HSA_MEM_FLAGS_NON_VOLATILE;
251
252 size_in_bytes =
253 ((uint64_t)mem->length_high << 32) +
254 mem->length_low;
255 width = mem->width;
256
257 /* Multiple banks of the same type are aggregated into
258 * one. User mode doesn't care about multiple physical
259 * memory segments. It's managed as a single virtual
260 * heap for user mode.
261 */
262 props = find_subtype_mem(heap_type, flags, width, dev);
263 if (props) {
264 props->size_in_bytes += size_in_bytes;
265 break;
266 }
267
268 props = kfd_alloc_struct(props);
269 if (!props)
270 return -ENOMEM;
271
272 props->heap_type = heap_type;
273 props->flags = flags;
274 props->size_in_bytes = size_in_bytes;
275 props->width = width;
276
277 dev->node_props.mem_banks_count++;
278 list_add_tail(&props->list, &dev->mem_props);
279
280 break;
281 }
282 }
283
284 return 0;
285 }
286
287 /* kfd_parse_subtype_cache - parse cache subtypes and attach it to correct
288 * topology device present in the device_list
289 */
kfd_parse_subtype_cache(struct crat_subtype_cache * cache,struct list_head * device_list)290 static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
291 struct list_head *device_list)
292 {
293 struct kfd_cache_properties *props;
294 struct kfd_topology_device *dev;
295 uint32_t id;
296 uint32_t total_num_of_cu;
297
298 id = cache->processor_id_low;
299
300 pr_debug("Found cache entry in CRAT table with processor_id=%d\n", id);
301 list_for_each_entry(dev, device_list, list) {
302 total_num_of_cu = (dev->node_props.array_count *
303 dev->node_props.cu_per_simd_array);
304
305 /* Cache infomration in CRAT doesn't have proximity_domain
306 * information as it is associated with a CPU core or GPU
307 * Compute Unit. So map the cache using CPU core Id or SIMD
308 * (GPU) ID.
309 * TODO: This works because currently we can safely assume that
310 * Compute Units are parsed before caches are parsed. In
311 * future, remove this dependency
312 */
313 if ((id >= dev->node_props.cpu_core_id_base &&
314 id <= dev->node_props.cpu_core_id_base +
315 dev->node_props.cpu_cores_count) ||
316 (id >= dev->node_props.simd_id_base &&
317 id < dev->node_props.simd_id_base +
318 total_num_of_cu)) {
319 props = kfd_alloc_struct(props);
320 if (!props)
321 return -ENOMEM;
322
323 props->processor_id_low = id;
324 props->cache_level = cache->cache_level;
325 props->cache_size = cache->cache_size;
326 props->cacheline_size = cache->cache_line_size;
327 props->cachelines_per_tag = cache->lines_per_tag;
328 props->cache_assoc = cache->associativity;
329 props->cache_latency = cache->cache_latency;
330 memcpy(props->sibling_map, cache->sibling_map,
331 sizeof(props->sibling_map));
332
333 if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
334 props->cache_type |= HSA_CACHE_TYPE_DATA;
335 if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
336 props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
337 if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
338 props->cache_type |= HSA_CACHE_TYPE_CPU;
339 if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
340 props->cache_type |= HSA_CACHE_TYPE_HSACU;
341
342 dev->cache_count++;
343 dev->node_props.caches_count++;
344 list_add_tail(&props->list, &dev->cache_props);
345
346 break;
347 }
348 }
349
350 return 0;
351 }
352
353 /* kfd_parse_subtype_iolink - parse iolink subtypes and attach it to correct
354 * topology device present in the device_list
355 */
kfd_parse_subtype_iolink(struct crat_subtype_iolink * iolink,struct list_head * device_list)356 static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
357 struct list_head *device_list)
358 {
359 struct kfd_iolink_properties *props = NULL, *props2;
360 struct kfd_topology_device *dev, *to_dev;
361 uint32_t id_from;
362 uint32_t id_to;
363
364 id_from = iolink->proximity_domain_from;
365 id_to = iolink->proximity_domain_to;
366
367 pr_debug("Found IO link entry in CRAT table with id_from=%d, id_to %d\n",
368 id_from, id_to);
369 list_for_each_entry(dev, device_list, list) {
370 if (id_from == dev->proximity_domain) {
371 props = kfd_alloc_struct(props);
372 if (!props)
373 return -ENOMEM;
374
375 props->node_from = id_from;
376 props->node_to = id_to;
377 props->ver_maj = iolink->version_major;
378 props->ver_min = iolink->version_minor;
379 props->iolink_type = iolink->io_interface_type;
380
381 if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
382 props->weight = 20;
383 else if (props->iolink_type == CRAT_IOLINK_TYPE_XGMI)
384 props->weight = 15 * iolink->num_hops_xgmi;
385 else
386 props->weight = node_distance(id_from, id_to);
387
388 props->min_latency = iolink->minimum_latency;
389 props->max_latency = iolink->maximum_latency;
390 props->min_bandwidth = iolink->minimum_bandwidth_mbs;
391 props->max_bandwidth = iolink->maximum_bandwidth_mbs;
392 props->rec_transfer_size =
393 iolink->recommended_transfer_size;
394
395 dev->io_link_count++;
396 dev->node_props.io_links_count++;
397 list_add_tail(&props->list, &dev->io_link_props);
398 break;
399 }
400 }
401
402 /* CPU topology is created before GPUs are detected, so CPU->GPU
403 * links are not built at that time. If a PCIe type is discovered, it
404 * means a GPU is detected and we are adding GPU->CPU to the topology.
405 * At this time, also add the corresponded CPU->GPU link if GPU
406 * is large bar.
407 * For xGMI, we only added the link with one direction in the crat
408 * table, add corresponded reversed direction link now.
409 */
410 if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
411 to_dev = kfd_topology_device_by_proximity_domain(id_to);
412 if (!to_dev)
413 return -ENODEV;
414 /* same everything but the other direction */
415 props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
416 props2->node_from = id_to;
417 props2->node_to = id_from;
418 props2->kobj = NULL;
419 to_dev->io_link_count++;
420 to_dev->node_props.io_links_count++;
421 list_add_tail(&props2->list, &to_dev->io_link_props);
422 }
423
424 return 0;
425 }
426
427 /* kfd_parse_subtype - parse subtypes and attach it to correct topology device
428 * present in the device_list
429 * @sub_type_hdr - subtype section of crat_image
430 * @device_list - list of topology devices present in this crat_image
431 */
kfd_parse_subtype(struct crat_subtype_generic * sub_type_hdr,struct list_head * device_list)432 static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr,
433 struct list_head *device_list)
434 {
435 struct crat_subtype_computeunit *cu;
436 struct crat_subtype_memory *mem;
437 struct crat_subtype_cache *cache;
438 struct crat_subtype_iolink *iolink;
439 int ret = 0;
440
441 switch (sub_type_hdr->type) {
442 case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
443 cu = (struct crat_subtype_computeunit *)sub_type_hdr;
444 ret = kfd_parse_subtype_cu(cu, device_list);
445 break;
446 case CRAT_SUBTYPE_MEMORY_AFFINITY:
447 mem = (struct crat_subtype_memory *)sub_type_hdr;
448 ret = kfd_parse_subtype_mem(mem, device_list);
449 break;
450 case CRAT_SUBTYPE_CACHE_AFFINITY:
451 cache = (struct crat_subtype_cache *)sub_type_hdr;
452 ret = kfd_parse_subtype_cache(cache, device_list);
453 break;
454 case CRAT_SUBTYPE_TLB_AFFINITY:
455 /*
456 * For now, nothing to do here
457 */
458 pr_debug("Found TLB entry in CRAT table (not processing)\n");
459 break;
460 case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
461 /*
462 * For now, nothing to do here
463 */
464 pr_debug("Found CCOMPUTE entry in CRAT table (not processing)\n");
465 break;
466 case CRAT_SUBTYPE_IOLINK_AFFINITY:
467 iolink = (struct crat_subtype_iolink *)sub_type_hdr;
468 ret = kfd_parse_subtype_iolink(iolink, device_list);
469 break;
470 default:
471 pr_warn("Unknown subtype %d in CRAT\n",
472 sub_type_hdr->type);
473 }
474
475 return ret;
476 }
477
478 /* kfd_parse_crat_table - parse CRAT table. For each node present in CRAT
479 * create a kfd_topology_device and add in to device_list. Also parse
480 * CRAT subtypes and attach it to appropriate kfd_topology_device
481 * @crat_image - input image containing CRAT
482 * @device_list - [OUT] list of kfd_topology_device generated after
483 * parsing crat_image
484 * @proximity_domain - Proximity domain of the first device in the table
485 *
486 * Return - 0 if successful else -ve value
487 */
kfd_parse_crat_table(void * crat_image,struct list_head * device_list,uint32_t proximity_domain)488 int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
489 uint32_t proximity_domain)
490 {
491 struct kfd_topology_device *top_dev = NULL;
492 struct crat_subtype_generic *sub_type_hdr;
493 uint16_t node_id;
494 int ret = 0;
495 struct crat_header *crat_table = (struct crat_header *)crat_image;
496 uint16_t num_nodes;
497 uint32_t image_len;
498
499 if (!crat_image)
500 return -EINVAL;
501
502 if (!list_empty(device_list)) {
503 pr_warn("Error device list should be empty\n");
504 return -EINVAL;
505 }
506
507 num_nodes = crat_table->num_domains;
508 image_len = crat_table->length;
509
510 pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
511
512 for (node_id = 0; node_id < num_nodes; node_id++) {
513 top_dev = kfd_create_topology_device(device_list);
514 if (!top_dev)
515 break;
516 top_dev->proximity_domain = proximity_domain++;
517 }
518
519 if (!top_dev) {
520 ret = -ENOMEM;
521 goto err;
522 }
523
524 memcpy(top_dev->oem_id, crat_table->oem_id, CRAT_OEMID_LENGTH);
525 memcpy(top_dev->oem_table_id, crat_table->oem_table_id,
526 CRAT_OEMTABLEID_LENGTH);
527 top_dev->oem_revision = crat_table->oem_revision;
528
529 sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
530 while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
531 ((char *)crat_image) + image_len) {
532 if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
533 ret = kfd_parse_subtype(sub_type_hdr, device_list);
534 if (ret)
535 break;
536 }
537
538 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
539 sub_type_hdr->length);
540 }
541
542 err:
543 if (ret)
544 kfd_release_topology_device_list(device_list);
545
546 return ret;
547 }
548
549 /* Helper function. See kfd_fill_gpu_cache_info for parameter description */
fill_in_pcache(struct crat_subtype_cache * pcache,struct kfd_gpu_cache_info * pcache_info,struct kfd_cu_info * cu_info,int mem_available,int cu_bitmask,int cache_type,unsigned int cu_processor_id,int cu_block)550 static int fill_in_pcache(struct crat_subtype_cache *pcache,
551 struct kfd_gpu_cache_info *pcache_info,
552 struct kfd_cu_info *cu_info,
553 int mem_available,
554 int cu_bitmask,
555 int cache_type, unsigned int cu_processor_id,
556 int cu_block)
557 {
558 unsigned int cu_sibling_map_mask;
559 int first_active_cu;
560
561 /* First check if enough memory is available */
562 if (sizeof(struct crat_subtype_cache) > mem_available)
563 return -ENOMEM;
564
565 cu_sibling_map_mask = cu_bitmask;
566 cu_sibling_map_mask >>= cu_block;
567 cu_sibling_map_mask &=
568 ((1 << pcache_info[cache_type].num_cu_shared) - 1);
569 first_active_cu = ffs(cu_sibling_map_mask);
570
571 /* CU could be inactive. In case of shared cache find the first active
572 * CU. and incase of non-shared cache check if the CU is inactive. If
573 * inactive active skip it
574 */
575 if (first_active_cu) {
576 memset(pcache, 0, sizeof(struct crat_subtype_cache));
577 pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
578 pcache->length = sizeof(struct crat_subtype_cache);
579 pcache->flags = pcache_info[cache_type].flags;
580 pcache->processor_id_low = cu_processor_id
581 + (first_active_cu - 1);
582 pcache->cache_level = pcache_info[cache_type].cache_level;
583 pcache->cache_size = pcache_info[cache_type].cache_size;
584
585 /* Sibling map is w.r.t processor_id_low, so shift out
586 * inactive CU
587 */
588 cu_sibling_map_mask =
589 cu_sibling_map_mask >> (first_active_cu - 1);
590
591 pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
592 pcache->sibling_map[1] =
593 (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
594 pcache->sibling_map[2] =
595 (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
596 pcache->sibling_map[3] =
597 (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
598 return 0;
599 }
600 return 1;
601 }
602
603 /* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
604 * tables
605 *
606 * @kdev - [IN] GPU device
607 * @gpu_processor_id - [IN] GPU processor ID to which these caches
608 * associate
609 * @available_size - [IN] Amount of memory available in pcache
610 * @cu_info - [IN] Compute Unit info obtained from KGD
611 * @pcache - [OUT] memory into which cache data is to be filled in.
612 * @size_filled - [OUT] amount of data used up in pcache.
613 * @num_of_entries - [OUT] number of caches added
614 */
kfd_fill_gpu_cache_info(struct kfd_dev * kdev,int gpu_processor_id,int available_size,struct kfd_cu_info * cu_info,struct crat_subtype_cache * pcache,int * size_filled,int * num_of_entries)615 static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
616 int gpu_processor_id,
617 int available_size,
618 struct kfd_cu_info *cu_info,
619 struct crat_subtype_cache *pcache,
620 int *size_filled,
621 int *num_of_entries)
622 {
623 struct kfd_gpu_cache_info *pcache_info;
624 int num_of_cache_types = 0;
625 int i, j, k;
626 int ct = 0;
627 int mem_available = available_size;
628 unsigned int cu_processor_id;
629 int ret;
630
631 switch (kdev->device_info->asic_family) {
632 case CHIP_KAVERI:
633 pcache_info = kaveri_cache_info;
634 num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
635 break;
636 case CHIP_HAWAII:
637 pcache_info = hawaii_cache_info;
638 num_of_cache_types = ARRAY_SIZE(hawaii_cache_info);
639 break;
640 case CHIP_CARRIZO:
641 pcache_info = carrizo_cache_info;
642 num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
643 break;
644 case CHIP_TONGA:
645 pcache_info = tonga_cache_info;
646 num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
647 break;
648 case CHIP_FIJI:
649 pcache_info = fiji_cache_info;
650 num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
651 break;
652 case CHIP_POLARIS10:
653 pcache_info = polaris10_cache_info;
654 num_of_cache_types = ARRAY_SIZE(polaris10_cache_info);
655 break;
656 case CHIP_POLARIS11:
657 pcache_info = polaris11_cache_info;
658 num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
659 break;
660 case CHIP_POLARIS12:
661 pcache_info = polaris12_cache_info;
662 num_of_cache_types = ARRAY_SIZE(polaris12_cache_info);
663 break;
664 case CHIP_VEGAM:
665 pcache_info = vegam_cache_info;
666 num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
667 break;
668 case CHIP_VEGA10:
669 case CHIP_VEGA12:
670 case CHIP_VEGA20:
671 case CHIP_ARCTURUS:
672 pcache_info = vega10_cache_info;
673 num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
674 break;
675 case CHIP_RAVEN:
676 pcache_info = raven_cache_info;
677 num_of_cache_types = ARRAY_SIZE(raven_cache_info);
678 break;
679 case CHIP_RENOIR:
680 pcache_info = renoir_cache_info;
681 num_of_cache_types = ARRAY_SIZE(renoir_cache_info);
682 break;
683 case CHIP_NAVI10:
684 case CHIP_NAVI12:
685 case CHIP_NAVI14:
686 pcache_info = navi10_cache_info;
687 num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
688 break;
689 default:
690 return -EINVAL;
691 }
692
693 *size_filled = 0;
694 *num_of_entries = 0;
695
696 /* For each type of cache listed in the kfd_gpu_cache_info table,
697 * go through all available Compute Units.
698 * The [i,j,k] loop will
699 * if kfd_gpu_cache_info.num_cu_shared = 1
700 * will parse through all available CU
701 * If (kfd_gpu_cache_info.num_cu_shared != 1)
702 * then it will consider only one CU from
703 * the shared unit
704 */
705
706 for (ct = 0; ct < num_of_cache_types; ct++) {
707 cu_processor_id = gpu_processor_id;
708 for (i = 0; i < cu_info->num_shader_engines; i++) {
709 for (j = 0; j < cu_info->num_shader_arrays_per_engine;
710 j++) {
711 for (k = 0; k < cu_info->num_cu_per_sh;
712 k += pcache_info[ct].num_cu_shared) {
713
714 ret = fill_in_pcache(pcache,
715 pcache_info,
716 cu_info,
717 mem_available,
718 cu_info->cu_bitmap[i % 4][j + i / 4],
719 ct,
720 cu_processor_id,
721 k);
722
723 if (ret < 0)
724 break;
725
726 if (!ret) {
727 pcache++;
728 (*num_of_entries)++;
729 mem_available -=
730 sizeof(*pcache);
731 (*size_filled) +=
732 sizeof(*pcache);
733 }
734
735 /* Move to next CU block */
736 cu_processor_id +=
737 pcache_info[ct].num_cu_shared;
738 }
739 }
740 }
741 }
742
743 pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
744
745 return 0;
746 }
747
748 /*
749 * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
750 * copies CRAT from ACPI (if available).
751 * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
752 *
753 * @crat_image: CRAT read from ACPI. If no CRAT in ACPI then
754 * crat_image will be NULL
755 * @size: [OUT] size of crat_image
756 *
757 * Return 0 if successful else return error code
758 */
kfd_create_crat_image_acpi(void ** crat_image,size_t * size)759 int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
760 {
761 struct acpi_table_header *crat_table;
762 acpi_status status;
763 void *pcrat_image;
764
765 if (!crat_image)
766 return -EINVAL;
767
768 *crat_image = NULL;
769
770 /* Fetch the CRAT table from ACPI */
771 status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
772 if (status == AE_NOT_FOUND) {
773 pr_warn("CRAT table not found\n");
774 return -ENODATA;
775 } else if (ACPI_FAILURE(status)) {
776 const char *err = acpi_format_exception(status);
777
778 pr_err("CRAT table error: %s\n", err);
779 return -EINVAL;
780 }
781
782 if (ignore_crat) {
783 pr_info("CRAT table disabled by module option\n");
784 return -ENODATA;
785 }
786
787 pcrat_image = kmemdup(crat_table, crat_table->length, GFP_KERNEL);
788 if (!pcrat_image)
789 return -ENOMEM;
790
791 *crat_image = pcrat_image;
792 *size = crat_table->length;
793
794 return 0;
795 }
796
797 /* Memory required to create Virtual CRAT.
798 * Since there is no easy way to predict the amount of memory required, the
799 * following amount are allocated for CPU and GPU Virtual CRAT. This is
800 * expected to cover all known conditions. But to be safe additional check
801 * is put in the code to ensure we don't overwrite.
802 */
803 #define VCRAT_SIZE_FOR_CPU (2 * PAGE_SIZE)
804 #define VCRAT_SIZE_FOR_GPU (4 * PAGE_SIZE)
805
806 /* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
807 *
808 * @numa_node_id: CPU NUMA node id
809 * @avail_size: Available size in the memory
810 * @sub_type_hdr: Memory into which compute info will be filled in
811 *
812 * Return 0 if successful else return -ve value
813 */
kfd_fill_cu_for_cpu(int numa_node_id,int * avail_size,int proximity_domain,struct crat_subtype_computeunit * sub_type_hdr)814 static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
815 int proximity_domain,
816 struct crat_subtype_computeunit *sub_type_hdr)
817 {
818 const struct cpumask *cpumask;
819
820 *avail_size -= sizeof(struct crat_subtype_computeunit);
821 if (*avail_size < 0)
822 return -ENOMEM;
823
824 memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
825
826 /* Fill in subtype header data */
827 sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
828 sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
829 sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
830
831 cpumask = cpumask_of_node(numa_node_id);
832
833 /* Fill in CU data */
834 sub_type_hdr->flags |= CRAT_CU_FLAGS_CPU_PRESENT;
835 sub_type_hdr->proximity_domain = proximity_domain;
836 sub_type_hdr->processor_id_low = kfd_numa_node_to_apic_id(numa_node_id);
837 if (sub_type_hdr->processor_id_low == -1)
838 return -EINVAL;
839
840 sub_type_hdr->num_cpu_cores = cpumask_weight(cpumask);
841
842 return 0;
843 }
844
845 /* kfd_fill_mem_info_for_cpu - Fill in Memory info for the given CPU NUMA node
846 *
847 * @numa_node_id: CPU NUMA node id
848 * @avail_size: Available size in the memory
849 * @sub_type_hdr: Memory into which compute info will be filled in
850 *
851 * Return 0 if successful else return -ve value
852 */
kfd_fill_mem_info_for_cpu(int numa_node_id,int * avail_size,int proximity_domain,struct crat_subtype_memory * sub_type_hdr)853 static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
854 int proximity_domain,
855 struct crat_subtype_memory *sub_type_hdr)
856 {
857 uint64_t mem_in_bytes = 0;
858 pg_data_t *pgdat;
859 int zone_type;
860
861 *avail_size -= sizeof(struct crat_subtype_memory);
862 if (*avail_size < 0)
863 return -ENOMEM;
864
865 memset(sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
866
867 /* Fill in subtype header data */
868 sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
869 sub_type_hdr->length = sizeof(struct crat_subtype_memory);
870 sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
871
872 /* Fill in Memory Subunit data */
873
874 /* Unlike si_meminfo, si_meminfo_node is not exported. So
875 * the following lines are duplicated from si_meminfo_node
876 * function
877 */
878 pgdat = NODE_DATA(numa_node_id);
879 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
880 mem_in_bytes += zone_managed_pages(&pgdat->node_zones[zone_type]);
881 mem_in_bytes <<= PAGE_SHIFT;
882
883 sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
884 sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
885 sub_type_hdr->proximity_domain = proximity_domain;
886
887 return 0;
888 }
889
890 #ifdef CONFIG_X86_64
kfd_fill_iolink_info_for_cpu(int numa_node_id,int * avail_size,uint32_t * num_entries,struct crat_subtype_iolink * sub_type_hdr)891 static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size,
892 uint32_t *num_entries,
893 struct crat_subtype_iolink *sub_type_hdr)
894 {
895 int nid;
896 struct cpuinfo_x86 *c = &cpu_data(0);
897 uint8_t link_type;
898
899 if (c->x86_vendor == X86_VENDOR_AMD)
900 link_type = CRAT_IOLINK_TYPE_HYPERTRANSPORT;
901 else
902 link_type = CRAT_IOLINK_TYPE_QPI_1_1;
903
904 *num_entries = 0;
905
906 /* Create IO links from this node to other CPU nodes */
907 for_each_online_node(nid) {
908 if (nid == numa_node_id) /* node itself */
909 continue;
910
911 *avail_size -= sizeof(struct crat_subtype_iolink);
912 if (*avail_size < 0)
913 return -ENOMEM;
914
915 memset(sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
916
917 /* Fill in subtype header data */
918 sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
919 sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
920 sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
921
922 /* Fill in IO link data */
923 sub_type_hdr->proximity_domain_from = numa_node_id;
924 sub_type_hdr->proximity_domain_to = nid;
925 sub_type_hdr->io_interface_type = link_type;
926
927 (*num_entries)++;
928 sub_type_hdr++;
929 }
930
931 return 0;
932 }
933 #endif
934
935 /* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
936 *
937 * @pcrat_image: Fill in VCRAT for CPU
938 * @size: [IN] allocated size of crat_image.
939 * [OUT] actual size of data filled in crat_image
940 */
kfd_create_vcrat_image_cpu(void * pcrat_image,size_t * size)941 static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
942 {
943 struct crat_header *crat_table = (struct crat_header *)pcrat_image;
944 struct acpi_table_header *acpi_table;
945 acpi_status status;
946 struct crat_subtype_generic *sub_type_hdr;
947 int avail_size = *size;
948 int numa_node_id;
949 #ifdef CONFIG_X86_64
950 uint32_t entries = 0;
951 #endif
952 int ret = 0;
953
954 if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_CPU)
955 return -EINVAL;
956
957 /* Fill in CRAT Header.
958 * Modify length and total_entries as subunits are added.
959 */
960 avail_size -= sizeof(struct crat_header);
961 if (avail_size < 0)
962 return -ENOMEM;
963
964 memset(crat_table, 0, sizeof(struct crat_header));
965 memcpy(&crat_table->signature, CRAT_SIGNATURE,
966 sizeof(crat_table->signature));
967 crat_table->length = sizeof(struct crat_header);
968
969 status = acpi_get_table("DSDT", 0, &acpi_table);
970 if (status != AE_OK)
971 pr_warn("DSDT table not found for OEM information\n");
972 else {
973 crat_table->oem_revision = acpi_table->revision;
974 memcpy(crat_table->oem_id, acpi_table->oem_id,
975 CRAT_OEMID_LENGTH);
976 memcpy(crat_table->oem_table_id, acpi_table->oem_table_id,
977 CRAT_OEMTABLEID_LENGTH);
978 }
979 crat_table->total_entries = 0;
980 crat_table->num_domains = 0;
981
982 sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
983
984 for_each_online_node(numa_node_id) {
985 if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
986 continue;
987
988 /* Fill in Subtype: Compute Unit */
989 ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
990 crat_table->num_domains,
991 (struct crat_subtype_computeunit *)sub_type_hdr);
992 if (ret < 0)
993 return ret;
994 crat_table->length += sub_type_hdr->length;
995 crat_table->total_entries++;
996
997 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
998 sub_type_hdr->length);
999
1000 /* Fill in Subtype: Memory */
1001 ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
1002 crat_table->num_domains,
1003 (struct crat_subtype_memory *)sub_type_hdr);
1004 if (ret < 0)
1005 return ret;
1006 crat_table->length += sub_type_hdr->length;
1007 crat_table->total_entries++;
1008
1009 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1010 sub_type_hdr->length);
1011
1012 /* Fill in Subtype: IO Link */
1013 #ifdef CONFIG_X86_64
1014 ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
1015 &entries,
1016 (struct crat_subtype_iolink *)sub_type_hdr);
1017 if (ret < 0)
1018 return ret;
1019 crat_table->length += (sub_type_hdr->length * entries);
1020 crat_table->total_entries += entries;
1021
1022 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1023 sub_type_hdr->length * entries);
1024 #else
1025 pr_info("IO link not available for non x86 platforms\n");
1026 #endif
1027
1028 crat_table->num_domains++;
1029 }
1030
1031 /* TODO: Add cache Subtype for CPU.
1032 * Currently, CPU cache information is available in function
1033 * detect_cache_attributes(cpu) defined in the file
1034 * ./arch/x86/kernel/cpu/intel_cacheinfo.c. This function is not
1035 * exported and to get the same information the code needs to be
1036 * duplicated.
1037 */
1038
1039 *size = crat_table->length;
1040 pr_info("Virtual CRAT table created for CPU\n");
1041
1042 return 0;
1043 }
1044
kfd_fill_gpu_memory_affinity(int * avail_size,struct kfd_dev * kdev,uint8_t type,uint64_t size,struct crat_subtype_memory * sub_type_hdr,uint32_t proximity_domain,const struct kfd_local_mem_info * local_mem_info)1045 static int kfd_fill_gpu_memory_affinity(int *avail_size,
1046 struct kfd_dev *kdev, uint8_t type, uint64_t size,
1047 struct crat_subtype_memory *sub_type_hdr,
1048 uint32_t proximity_domain,
1049 const struct kfd_local_mem_info *local_mem_info)
1050 {
1051 *avail_size -= sizeof(struct crat_subtype_memory);
1052 if (*avail_size < 0)
1053 return -ENOMEM;
1054
1055 memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
1056 sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
1057 sub_type_hdr->length = sizeof(struct crat_subtype_memory);
1058 sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1059
1060 sub_type_hdr->proximity_domain = proximity_domain;
1061
1062 pr_debug("Fill gpu memory affinity - type 0x%x size 0x%llx\n",
1063 type, size);
1064
1065 sub_type_hdr->length_low = lower_32_bits(size);
1066 sub_type_hdr->length_high = upper_32_bits(size);
1067
1068 sub_type_hdr->width = local_mem_info->vram_width;
1069 sub_type_hdr->visibility_type = type;
1070
1071 return 0;
1072 }
1073
1074 /* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
1075 * to its NUMA node
1076 * @avail_size: Available size in the memory
1077 * @kdev - [IN] GPU device
1078 * @sub_type_hdr: Memory into which io link info will be filled in
1079 * @proximity_domain - proximity domain of the GPU node
1080 *
1081 * Return 0 if successful else return -ve value
1082 */
kfd_fill_gpu_direct_io_link_to_cpu(int * avail_size,struct kfd_dev * kdev,struct crat_subtype_iolink * sub_type_hdr,uint32_t proximity_domain)1083 static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
1084 struct kfd_dev *kdev,
1085 struct crat_subtype_iolink *sub_type_hdr,
1086 uint32_t proximity_domain)
1087 {
1088 *avail_size -= sizeof(struct crat_subtype_iolink);
1089 if (*avail_size < 0)
1090 return -ENOMEM;
1091
1092 memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1093
1094 /* Fill in subtype header data */
1095 sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1096 sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1097 sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1098 if (kfd_dev_is_large_bar(kdev))
1099 sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1100
1101 /* Fill in IOLINK subtype.
1102 * TODO: Fill-in other fields of iolink subtype
1103 */
1104 sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
1105 sub_type_hdr->proximity_domain_from = proximity_domain;
1106 #ifdef CONFIG_NUMA
1107 if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
1108 sub_type_hdr->proximity_domain_to = 0;
1109 else
1110 sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node;
1111 #else
1112 sub_type_hdr->proximity_domain_to = 0;
1113 #endif
1114 return 0;
1115 }
1116
kfd_fill_gpu_xgmi_link_to_gpu(int * avail_size,struct kfd_dev * kdev,struct kfd_dev * peer_kdev,struct crat_subtype_iolink * sub_type_hdr,uint32_t proximity_domain_from,uint32_t proximity_domain_to)1117 static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
1118 struct kfd_dev *kdev,
1119 struct kfd_dev *peer_kdev,
1120 struct crat_subtype_iolink *sub_type_hdr,
1121 uint32_t proximity_domain_from,
1122 uint32_t proximity_domain_to)
1123 {
1124 *avail_size -= sizeof(struct crat_subtype_iolink);
1125 if (*avail_size < 0)
1126 return -ENOMEM;
1127
1128 memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1129
1130 sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1131 sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1132 sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED |
1133 CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1134
1135 sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
1136 sub_type_hdr->proximity_domain_from = proximity_domain_from;
1137 sub_type_hdr->proximity_domain_to = proximity_domain_to;
1138 sub_type_hdr->num_hops_xgmi =
1139 amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
1140 return 0;
1141 }
1142
1143 /* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
1144 *
1145 * @pcrat_image: Fill in VCRAT for GPU
1146 * @size: [IN] allocated size of crat_image.
1147 * [OUT] actual size of data filled in crat_image
1148 */
kfd_create_vcrat_image_gpu(void * pcrat_image,size_t * size,struct kfd_dev * kdev,uint32_t proximity_domain)1149 static int kfd_create_vcrat_image_gpu(void *pcrat_image,
1150 size_t *size, struct kfd_dev *kdev,
1151 uint32_t proximity_domain)
1152 {
1153 struct crat_header *crat_table = (struct crat_header *)pcrat_image;
1154 struct crat_subtype_generic *sub_type_hdr;
1155 struct kfd_local_mem_info local_mem_info;
1156 struct kfd_topology_device *peer_dev;
1157 struct crat_subtype_computeunit *cu;
1158 struct kfd_cu_info cu_info;
1159 int avail_size = *size;
1160 uint32_t total_num_of_cu;
1161 int num_of_cache_entries = 0;
1162 int cache_mem_filled = 0;
1163 uint32_t nid = 0;
1164 int ret = 0;
1165
1166 if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
1167 return -EINVAL;
1168
1169 /* Fill the CRAT Header.
1170 * Modify length and total_entries as subunits are added.
1171 */
1172 avail_size -= sizeof(struct crat_header);
1173 if (avail_size < 0)
1174 return -ENOMEM;
1175
1176 memset(crat_table, 0, sizeof(struct crat_header));
1177
1178 memcpy(&crat_table->signature, CRAT_SIGNATURE,
1179 sizeof(crat_table->signature));
1180 /* Change length as we add more subtypes*/
1181 crat_table->length = sizeof(struct crat_header);
1182 crat_table->num_domains = 1;
1183 crat_table->total_entries = 0;
1184
1185 /* Fill in Subtype: Compute Unit
1186 * First fill in the sub type header and then sub type data
1187 */
1188 avail_size -= sizeof(struct crat_subtype_computeunit);
1189 if (avail_size < 0)
1190 return -ENOMEM;
1191
1192 sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
1193 memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
1194
1195 sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
1196 sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
1197 sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
1198
1199 /* Fill CU subtype data */
1200 cu = (struct crat_subtype_computeunit *)sub_type_hdr;
1201 cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
1202 cu->proximity_domain = proximity_domain;
1203
1204 amdgpu_amdkfd_get_cu_info(kdev->kgd, &cu_info);
1205 cu->num_simd_per_cu = cu_info.simd_per_cu;
1206 cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
1207 cu->max_waves_simd = cu_info.max_waves_per_simd;
1208
1209 cu->wave_front_size = cu_info.wave_front_size;
1210 cu->array_count = cu_info.num_shader_arrays_per_engine *
1211 cu_info.num_shader_engines;
1212 total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
1213 cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
1214 cu->num_cu_per_array = cu_info.num_cu_per_sh;
1215 cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
1216 cu->num_banks = cu_info.num_shader_engines;
1217 cu->lds_size_in_kb = cu_info.lds_size;
1218
1219 cu->hsa_capability = 0;
1220
1221 /* Check if this node supports IOMMU. During parsing this flag will
1222 * translate to HSA_CAP_ATS_PRESENT
1223 */
1224 if (!kfd_iommu_check_device(kdev))
1225 cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
1226
1227 crat_table->length += sub_type_hdr->length;
1228 crat_table->total_entries++;
1229
1230 /* Fill in Subtype: Memory. Only on systems with large BAR (no
1231 * private FB), report memory as public. On other systems
1232 * report the total FB size (public+private) as a single
1233 * private heap.
1234 */
1235 amdgpu_amdkfd_get_local_mem_info(kdev->kgd, &local_mem_info);
1236 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1237 sub_type_hdr->length);
1238
1239 if (debug_largebar)
1240 local_mem_info.local_mem_size_private = 0;
1241
1242 if (local_mem_info.local_mem_size_private == 0)
1243 ret = kfd_fill_gpu_memory_affinity(&avail_size,
1244 kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
1245 local_mem_info.local_mem_size_public,
1246 (struct crat_subtype_memory *)sub_type_hdr,
1247 proximity_domain,
1248 &local_mem_info);
1249 else
1250 ret = kfd_fill_gpu_memory_affinity(&avail_size,
1251 kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
1252 local_mem_info.local_mem_size_public +
1253 local_mem_info.local_mem_size_private,
1254 (struct crat_subtype_memory *)sub_type_hdr,
1255 proximity_domain,
1256 &local_mem_info);
1257 if (ret < 0)
1258 return ret;
1259
1260 crat_table->length += sizeof(struct crat_subtype_memory);
1261 crat_table->total_entries++;
1262
1263 /* TODO: Fill in cache information. This information is NOT readily
1264 * available in KGD
1265 */
1266 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1267 sub_type_hdr->length);
1268 ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
1269 avail_size,
1270 &cu_info,
1271 (struct crat_subtype_cache *)sub_type_hdr,
1272 &cache_mem_filled,
1273 &num_of_cache_entries);
1274
1275 if (ret < 0)
1276 return ret;
1277
1278 crat_table->length += cache_mem_filled;
1279 crat_table->total_entries += num_of_cache_entries;
1280 avail_size -= cache_mem_filled;
1281
1282 /* Fill in Subtype: IO_LINKS
1283 * Only direct links are added here which is Link from GPU to
1284 * to its NUMA node. Indirect links are added by userspace.
1285 */
1286 sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1287 cache_mem_filled);
1288 ret = kfd_fill_gpu_direct_io_link_to_cpu(&avail_size, kdev,
1289 (struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
1290
1291 if (ret < 0)
1292 return ret;
1293
1294 crat_table->length += sub_type_hdr->length;
1295 crat_table->total_entries++;
1296
1297
1298 /* Fill in Subtype: IO_LINKS
1299 * Direct links from GPU to other GPUs through xGMI.
1300 * We will loop GPUs that already be processed (with lower value
1301 * of proximity_domain), add the link for the GPUs with same
1302 * hive id (from this GPU to other GPU) . The reversed iolink
1303 * (from other GPU to this GPU) will be added
1304 * in kfd_parse_subtype_iolink.
1305 */
1306 if (kdev->hive_id) {
1307 for (nid = 0; nid < proximity_domain; ++nid) {
1308 peer_dev = kfd_topology_device_by_proximity_domain(nid);
1309 if (!peer_dev->gpu)
1310 continue;
1311 if (peer_dev->gpu->hive_id != kdev->hive_id)
1312 continue;
1313 sub_type_hdr = (typeof(sub_type_hdr))(
1314 (char *)sub_type_hdr +
1315 sizeof(struct crat_subtype_iolink));
1316 ret = kfd_fill_gpu_xgmi_link_to_gpu(
1317 &avail_size, kdev, peer_dev->gpu,
1318 (struct crat_subtype_iolink *)sub_type_hdr,
1319 proximity_domain, nid);
1320 if (ret < 0)
1321 return ret;
1322 crat_table->length += sub_type_hdr->length;
1323 crat_table->total_entries++;
1324 }
1325 }
1326 *size = crat_table->length;
1327 pr_info("Virtual CRAT table created for GPU\n");
1328
1329 return ret;
1330 }
1331
1332 /* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
1333 * creates a Virtual CRAT (VCRAT) image
1334 *
1335 * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
1336 *
1337 * @crat_image: VCRAT image created because ACPI does not have a
1338 * CRAT for this device
1339 * @size: [OUT] size of virtual crat_image
1340 * @flags: COMPUTE_UNIT_CPU - Create VCRAT for CPU device
1341 * COMPUTE_UNIT_GPU - Create VCRAT for GPU
1342 * (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
1343 * -- this option is not currently implemented.
1344 * The assumption is that all AMD APUs will have CRAT
1345 * @kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
1346 *
1347 * Return 0 if successful else return -ve value
1348 */
kfd_create_crat_image_virtual(void ** crat_image,size_t * size,int flags,struct kfd_dev * kdev,uint32_t proximity_domain)1349 int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
1350 int flags, struct kfd_dev *kdev,
1351 uint32_t proximity_domain)
1352 {
1353 void *pcrat_image = NULL;
1354 int ret = 0;
1355
1356 if (!crat_image)
1357 return -EINVAL;
1358
1359 *crat_image = NULL;
1360
1361 /* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and
1362 * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover
1363 * all the current conditions. A check is put not to overwrite beyond
1364 * allocated size
1365 */
1366 switch (flags) {
1367 case COMPUTE_UNIT_CPU:
1368 pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL);
1369 if (!pcrat_image)
1370 return -ENOMEM;
1371 *size = VCRAT_SIZE_FOR_CPU;
1372 ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
1373 break;
1374 case COMPUTE_UNIT_GPU:
1375 if (!kdev)
1376 return -EINVAL;
1377 pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
1378 if (!pcrat_image)
1379 return -ENOMEM;
1380 *size = VCRAT_SIZE_FOR_GPU;
1381 ret = kfd_create_vcrat_image_gpu(pcrat_image, size, kdev,
1382 proximity_domain);
1383 break;
1384 case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU):
1385 /* TODO: */
1386 ret = -EINVAL;
1387 pr_err("VCRAT not implemented for APU\n");
1388 break;
1389 default:
1390 ret = -EINVAL;
1391 }
1392
1393 if (!ret)
1394 *crat_image = pcrat_image;
1395 else
1396 kfree(pcrat_image);
1397
1398 return ret;
1399 }
1400
1401
1402 /* kfd_destroy_crat_image
1403 *
1404 * @crat_image: [IN] - crat_image from kfd_create_crat_image_xxx(..)
1405 *
1406 */
kfd_destroy_crat_image(void * crat_image)1407 void kfd_destroy_crat_image(void *crat_image)
1408 {
1409 kfree(crat_image);
1410 }
1411