1 /* $NetBSD: kfd_topology.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $ */
2
3 /*
4 * Copyright 2014 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(0, "$NetBSD: kfd_topology.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $");
27
28 #include <linux/types.h>
29 #include <linux/kernel.h>
30 #include <linux/pci.h>
31 #include <linux/errno.h>
32 #include <linux/acpi.h>
33 #include <linux/hash.h>
34 #include <linux/cpufreq.h>
35 #include <linux/log2.h>
36 #include <linux/dmi.h>
37 #include <linux/atomic.h>
38
39 #include "kfd_priv.h"
40 #include "kfd_crat.h"
41 #include "kfd_topology.h"
42 #include "kfd_device_queue_manager.h"
43 #include "kfd_iommu.h"
44 #include "amdgpu_amdkfd.h"
45 #include "amdgpu_ras.h"
46
47 /* topology_device_list - Master list of all topology devices */
48 static struct list_head topology_device_list;
49 static struct kfd_system_properties sys_props;
50
51 static DECLARE_RWSEM(topology_lock);
52 static atomic_t topology_crat_proximity_domain;
53
kfd_topology_device_by_proximity_domain(uint32_t proximity_domain)54 struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
55 uint32_t proximity_domain)
56 {
57 struct kfd_topology_device *top_dev;
58 struct kfd_topology_device *device = NULL;
59
60 down_read(&topology_lock);
61
62 list_for_each_entry(top_dev, &topology_device_list, list)
63 if (top_dev->proximity_domain == proximity_domain) {
64 device = top_dev;
65 break;
66 }
67
68 up_read(&topology_lock);
69
70 return device;
71 }
72
kfd_topology_device_by_id(uint32_t gpu_id)73 struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id)
74 {
75 struct kfd_topology_device *top_dev = NULL;
76 struct kfd_topology_device *ret = NULL;
77
78 down_read(&topology_lock);
79
80 list_for_each_entry(top_dev, &topology_device_list, list)
81 if (top_dev->gpu_id == gpu_id) {
82 ret = top_dev;
83 break;
84 }
85
86 up_read(&topology_lock);
87
88 return ret;
89 }
90
kfd_device_by_id(uint32_t gpu_id)91 struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
92 {
93 struct kfd_topology_device *top_dev;
94
95 top_dev = kfd_topology_device_by_id(gpu_id);
96 if (!top_dev)
97 return NULL;
98
99 return top_dev->gpu;
100 }
101
kfd_device_by_pci_dev(const struct pci_dev * pdev)102 struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
103 {
104 struct kfd_topology_device *top_dev;
105 struct kfd_dev *device = NULL;
106
107 down_read(&topology_lock);
108
109 list_for_each_entry(top_dev, &topology_device_list, list)
110 if (top_dev->gpu && top_dev->gpu->pdev == pdev) {
111 device = top_dev->gpu;
112 break;
113 }
114
115 up_read(&topology_lock);
116
117 return device;
118 }
119
kfd_device_by_kgd(const struct kgd_dev * kgd)120 struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd)
121 {
122 struct kfd_topology_device *top_dev;
123 struct kfd_dev *device = NULL;
124
125 down_read(&topology_lock);
126
127 list_for_each_entry(top_dev, &topology_device_list, list)
128 if (top_dev->gpu && top_dev->gpu->kgd == kgd) {
129 device = top_dev->gpu;
130 break;
131 }
132
133 up_read(&topology_lock);
134
135 return device;
136 }
137
138 /* Called with write topology_lock acquired */
kfd_release_topology_device(struct kfd_topology_device * dev)139 static void kfd_release_topology_device(struct kfd_topology_device *dev)
140 {
141 struct kfd_mem_properties *mem;
142 struct kfd_cache_properties *cache;
143 struct kfd_iolink_properties *iolink;
144 struct kfd_perf_properties *perf;
145
146 list_del(&dev->list);
147
148 while (dev->mem_props.next != &dev->mem_props) {
149 mem = container_of(dev->mem_props.next,
150 struct kfd_mem_properties, list);
151 list_del(&mem->list);
152 kfree(mem);
153 }
154
155 while (dev->cache_props.next != &dev->cache_props) {
156 cache = container_of(dev->cache_props.next,
157 struct kfd_cache_properties, list);
158 list_del(&cache->list);
159 kfree(cache);
160 }
161
162 while (dev->io_link_props.next != &dev->io_link_props) {
163 iolink = container_of(dev->io_link_props.next,
164 struct kfd_iolink_properties, list);
165 list_del(&iolink->list);
166 kfree(iolink);
167 }
168
169 while (dev->perf_props.next != &dev->perf_props) {
170 perf = container_of(dev->perf_props.next,
171 struct kfd_perf_properties, list);
172 list_del(&perf->list);
173 kfree(perf);
174 }
175
176 kfree(dev);
177 }
178
kfd_release_topology_device_list(struct list_head * device_list)179 void kfd_release_topology_device_list(struct list_head *device_list)
180 {
181 struct kfd_topology_device *dev;
182
183 while (!list_empty(device_list)) {
184 dev = list_first_entry(device_list,
185 struct kfd_topology_device, list);
186 kfd_release_topology_device(dev);
187 }
188 }
189
kfd_release_live_view(void)190 static void kfd_release_live_view(void)
191 {
192 kfd_release_topology_device_list(&topology_device_list);
193 memset(&sys_props, 0, sizeof(sys_props));
194 }
195
kfd_create_topology_device(struct list_head * device_list)196 struct kfd_topology_device *kfd_create_topology_device(
197 struct list_head *device_list)
198 {
199 struct kfd_topology_device *dev;
200
201 dev = kfd_alloc_struct(dev);
202 if (!dev) {
203 pr_err("No memory to allocate a topology device");
204 return NULL;
205 }
206
207 INIT_LIST_HEAD(&dev->mem_props);
208 INIT_LIST_HEAD(&dev->cache_props);
209 INIT_LIST_HEAD(&dev->io_link_props);
210 INIT_LIST_HEAD(&dev->perf_props);
211
212 list_add_tail(&dev->list, device_list);
213
214 return dev;
215 }
216
217
218 #define sysfs_show_gen_prop(buffer, fmt, ...) \
219 snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__)
220 #define sysfs_show_32bit_prop(buffer, name, value) \
221 sysfs_show_gen_prop(buffer, "%s %u\n", name, value)
222 #define sysfs_show_64bit_prop(buffer, name, value) \
223 sysfs_show_gen_prop(buffer, "%s %llu\n", name, value)
224 #define sysfs_show_32bit_val(buffer, value) \
225 sysfs_show_gen_prop(buffer, "%u\n", value)
226 #define sysfs_show_str_val(buffer, value) \
227 sysfs_show_gen_prop(buffer, "%s\n", value)
228
sysprops_show(struct kobject * kobj,struct attribute * attr,char * buffer)229 static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
230 char *buffer)
231 {
232 ssize_t ret;
233
234 /* Making sure that the buffer is an empty string */
235 buffer[0] = 0;
236
237 if (attr == &sys_props.attr_genid) {
238 ret = sysfs_show_32bit_val(buffer, sys_props.generation_count);
239 } else if (attr == &sys_props.attr_props) {
240 sysfs_show_64bit_prop(buffer, "platform_oem",
241 sys_props.platform_oem);
242 sysfs_show_64bit_prop(buffer, "platform_id",
243 sys_props.platform_id);
244 ret = sysfs_show_64bit_prop(buffer, "platform_rev",
245 sys_props.platform_rev);
246 } else {
247 ret = -EINVAL;
248 }
249
250 return ret;
251 }
252
kfd_topology_kobj_release(struct kobject * kobj)253 static void kfd_topology_kobj_release(struct kobject *kobj)
254 {
255 kfree(kobj);
256 }
257
258 static const struct sysfs_ops sysprops_ops = {
259 .show = sysprops_show,
260 };
261
262 static struct kobj_type sysprops_type = {
263 .release = kfd_topology_kobj_release,
264 .sysfs_ops = &sysprops_ops,
265 };
266
iolink_show(struct kobject * kobj,struct attribute * attr,char * buffer)267 static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr,
268 char *buffer)
269 {
270 ssize_t ret;
271 struct kfd_iolink_properties *iolink;
272
273 /* Making sure that the buffer is an empty string */
274 buffer[0] = 0;
275
276 iolink = container_of(attr, struct kfd_iolink_properties, attr);
277 if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu))
278 return -EPERM;
279 sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type);
280 sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj);
281 sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min);
282 sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from);
283 sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to);
284 sysfs_show_32bit_prop(buffer, "weight", iolink->weight);
285 sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency);
286 sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency);
287 sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth);
288 sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth);
289 sysfs_show_32bit_prop(buffer, "recommended_transfer_size",
290 iolink->rec_transfer_size);
291 ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags);
292
293 return ret;
294 }
295
296 static const struct sysfs_ops iolink_ops = {
297 .show = iolink_show,
298 };
299
300 static struct kobj_type iolink_type = {
301 .release = kfd_topology_kobj_release,
302 .sysfs_ops = &iolink_ops,
303 };
304
mem_show(struct kobject * kobj,struct attribute * attr,char * buffer)305 static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
306 char *buffer)
307 {
308 ssize_t ret;
309 struct kfd_mem_properties *mem;
310
311 /* Making sure that the buffer is an empty string */
312 buffer[0] = 0;
313
314 mem = container_of(attr, struct kfd_mem_properties, attr);
315 if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu))
316 return -EPERM;
317 sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type);
318 sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes);
319 sysfs_show_32bit_prop(buffer, "flags", mem->flags);
320 sysfs_show_32bit_prop(buffer, "width", mem->width);
321 ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max);
322
323 return ret;
324 }
325
326 static const struct sysfs_ops mem_ops = {
327 .show = mem_show,
328 };
329
330 static struct kobj_type mem_type = {
331 .release = kfd_topology_kobj_release,
332 .sysfs_ops = &mem_ops,
333 };
334
kfd_cache_show(struct kobject * kobj,struct attribute * attr,char * buffer)335 static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
336 char *buffer)
337 {
338 ssize_t ret;
339 uint32_t i, j;
340 struct kfd_cache_properties *cache;
341
342 /* Making sure that the buffer is an empty string */
343 buffer[0] = 0;
344
345 cache = container_of(attr, struct kfd_cache_properties, attr);
346 if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu))
347 return -EPERM;
348 sysfs_show_32bit_prop(buffer, "processor_id_low",
349 cache->processor_id_low);
350 sysfs_show_32bit_prop(buffer, "level", cache->cache_level);
351 sysfs_show_32bit_prop(buffer, "size", cache->cache_size);
352 sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size);
353 sysfs_show_32bit_prop(buffer, "cache_lines_per_tag",
354 cache->cachelines_per_tag);
355 sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc);
356 sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency);
357 sysfs_show_32bit_prop(buffer, "type", cache->cache_type);
358 snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer);
359 for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++)
360 for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) {
361 /* Check each bit */
362 if (cache->sibling_map[i] & (1 << j))
363 ret = snprintf(buffer, PAGE_SIZE,
364 "%s%d%s", buffer, 1, ",");
365 else
366 ret = snprintf(buffer, PAGE_SIZE,
367 "%s%d%s", buffer, 0, ",");
368 }
369 /* Replace the last "," with end of line */
370 *(buffer + strlen(buffer) - 1) = 0xA;
371 return ret;
372 }
373
374 static const struct sysfs_ops cache_ops = {
375 .show = kfd_cache_show,
376 };
377
378 static struct kobj_type cache_type = {
379 .release = kfd_topology_kobj_release,
380 .sysfs_ops = &cache_ops,
381 };
382
383 /****** Sysfs of Performance Counters ******/
384
385 struct kfd_perf_attr {
386 struct kobj_attribute attr;
387 uint32_t data;
388 };
389
perf_show(struct kobject * kobj,struct kobj_attribute * attrs,char * buf)390 static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs,
391 char *buf)
392 {
393 struct kfd_perf_attr *attr;
394
395 buf[0] = 0;
396 attr = container_of(attrs, struct kfd_perf_attr, attr);
397 if (!attr->data) /* invalid data for PMC */
398 return 0;
399 else
400 return sysfs_show_32bit_val(buf, attr->data);
401 }
402
403 #define KFD_PERF_DESC(_name, _data) \
404 { \
405 .attr = __ATTR(_name, 0444, perf_show, NULL), \
406 .data = _data, \
407 }
408
409 static struct kfd_perf_attr perf_attr_iommu[] = {
410 KFD_PERF_DESC(max_concurrent, 0),
411 KFD_PERF_DESC(num_counters, 0),
412 KFD_PERF_DESC(counter_ids, 0),
413 };
414 /****************************************/
415
node_show(struct kobject * kobj,struct attribute * attr,char * buffer)416 static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
417 char *buffer)
418 {
419 struct kfd_topology_device *dev;
420 uint32_t log_max_watch_addr;
421
422 /* Making sure that the buffer is an empty string */
423 buffer[0] = 0;
424
425 if (strcmp(attr->name, "gpu_id") == 0) {
426 dev = container_of(attr, struct kfd_topology_device,
427 attr_gpuid);
428 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
429 return -EPERM;
430 return sysfs_show_32bit_val(buffer, dev->gpu_id);
431 }
432
433 if (strcmp(attr->name, "name") == 0) {
434 dev = container_of(attr, struct kfd_topology_device,
435 attr_name);
436
437 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
438 return -EPERM;
439 return sysfs_show_str_val(buffer, dev->node_props.name);
440 }
441
442 dev = container_of(attr, struct kfd_topology_device,
443 attr_props);
444 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
445 return -EPERM;
446 sysfs_show_32bit_prop(buffer, "cpu_cores_count",
447 dev->node_props.cpu_cores_count);
448 sysfs_show_32bit_prop(buffer, "simd_count",
449 dev->node_props.simd_count);
450 sysfs_show_32bit_prop(buffer, "mem_banks_count",
451 dev->node_props.mem_banks_count);
452 sysfs_show_32bit_prop(buffer, "caches_count",
453 dev->node_props.caches_count);
454 sysfs_show_32bit_prop(buffer, "io_links_count",
455 dev->node_props.io_links_count);
456 sysfs_show_32bit_prop(buffer, "cpu_core_id_base",
457 dev->node_props.cpu_core_id_base);
458 sysfs_show_32bit_prop(buffer, "simd_id_base",
459 dev->node_props.simd_id_base);
460 sysfs_show_32bit_prop(buffer, "max_waves_per_simd",
461 dev->node_props.max_waves_per_simd);
462 sysfs_show_32bit_prop(buffer, "lds_size_in_kb",
463 dev->node_props.lds_size_in_kb);
464 sysfs_show_32bit_prop(buffer, "gds_size_in_kb",
465 dev->node_props.gds_size_in_kb);
466 sysfs_show_32bit_prop(buffer, "num_gws",
467 dev->node_props.num_gws);
468 sysfs_show_32bit_prop(buffer, "wave_front_size",
469 dev->node_props.wave_front_size);
470 sysfs_show_32bit_prop(buffer, "array_count",
471 dev->node_props.array_count);
472 sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine",
473 dev->node_props.simd_arrays_per_engine);
474 sysfs_show_32bit_prop(buffer, "cu_per_simd_array",
475 dev->node_props.cu_per_simd_array);
476 sysfs_show_32bit_prop(buffer, "simd_per_cu",
477 dev->node_props.simd_per_cu);
478 sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu",
479 dev->node_props.max_slots_scratch_cu);
480 sysfs_show_32bit_prop(buffer, "vendor_id",
481 dev->node_props.vendor_id);
482 sysfs_show_32bit_prop(buffer, "device_id",
483 dev->node_props.device_id);
484 sysfs_show_32bit_prop(buffer, "location_id",
485 dev->node_props.location_id);
486 sysfs_show_32bit_prop(buffer, "drm_render_minor",
487 dev->node_props.drm_render_minor);
488 sysfs_show_64bit_prop(buffer, "hive_id",
489 dev->node_props.hive_id);
490 sysfs_show_32bit_prop(buffer, "num_sdma_engines",
491 dev->node_props.num_sdma_engines);
492 sysfs_show_32bit_prop(buffer, "num_sdma_xgmi_engines",
493 dev->node_props.num_sdma_xgmi_engines);
494 sysfs_show_32bit_prop(buffer, "num_sdma_queues_per_engine",
495 dev->node_props.num_sdma_queues_per_engine);
496 sysfs_show_32bit_prop(buffer, "num_cp_queues",
497 dev->node_props.num_cp_queues);
498
499 if (dev->gpu) {
500 log_max_watch_addr =
501 __ilog2_u32(dev->gpu->device_info->num_of_watch_points);
502
503 if (log_max_watch_addr) {
504 dev->node_props.capability |=
505 HSA_CAP_WATCH_POINTS_SUPPORTED;
506
507 dev->node_props.capability |=
508 ((log_max_watch_addr <<
509 HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) &
510 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
511 }
512
513 if (dev->gpu->device_info->asic_family == CHIP_TONGA)
514 dev->node_props.capability |=
515 HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
516
517 sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
518 dev->node_props.max_engine_clk_fcompute);
519
520 sysfs_show_64bit_prop(buffer, "local_mem_size",
521 (unsigned long long int) 0);
522
523 sysfs_show_32bit_prop(buffer, "fw_version",
524 dev->gpu->mec_fw_version);
525 sysfs_show_32bit_prop(buffer, "capability",
526 dev->node_props.capability);
527 sysfs_show_32bit_prop(buffer, "sdma_fw_version",
528 dev->gpu->sdma_fw_version);
529 }
530
531 return sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute",
532 cpufreq_quick_get_max(0)/1000);
533 }
534
535 static const struct sysfs_ops node_ops = {
536 .show = node_show,
537 };
538
539 static struct kobj_type node_type = {
540 .release = kfd_topology_kobj_release,
541 .sysfs_ops = &node_ops,
542 };
543
kfd_remove_sysfs_file(struct kobject * kobj,struct attribute * attr)544 static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr)
545 {
546 sysfs_remove_file(kobj, attr);
547 kobject_del(kobj);
548 kobject_put(kobj);
549 }
550
kfd_remove_sysfs_node_entry(struct kfd_topology_device * dev)551 static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
552 {
553 struct kfd_iolink_properties *iolink;
554 struct kfd_cache_properties *cache;
555 struct kfd_mem_properties *mem;
556 struct kfd_perf_properties *perf;
557
558 if (dev->kobj_iolink) {
559 list_for_each_entry(iolink, &dev->io_link_props, list)
560 if (iolink->kobj) {
561 kfd_remove_sysfs_file(iolink->kobj,
562 &iolink->attr);
563 iolink->kobj = NULL;
564 }
565 kobject_del(dev->kobj_iolink);
566 kobject_put(dev->kobj_iolink);
567 dev->kobj_iolink = NULL;
568 }
569
570 if (dev->kobj_cache) {
571 list_for_each_entry(cache, &dev->cache_props, list)
572 if (cache->kobj) {
573 kfd_remove_sysfs_file(cache->kobj,
574 &cache->attr);
575 cache->kobj = NULL;
576 }
577 kobject_del(dev->kobj_cache);
578 kobject_put(dev->kobj_cache);
579 dev->kobj_cache = NULL;
580 }
581
582 if (dev->kobj_mem) {
583 list_for_each_entry(mem, &dev->mem_props, list)
584 if (mem->kobj) {
585 kfd_remove_sysfs_file(mem->kobj, &mem->attr);
586 mem->kobj = NULL;
587 }
588 kobject_del(dev->kobj_mem);
589 kobject_put(dev->kobj_mem);
590 dev->kobj_mem = NULL;
591 }
592
593 if (dev->kobj_perf) {
594 list_for_each_entry(perf, &dev->perf_props, list) {
595 kfree(perf->attr_group);
596 perf->attr_group = NULL;
597 }
598 kobject_del(dev->kobj_perf);
599 kobject_put(dev->kobj_perf);
600 dev->kobj_perf = NULL;
601 }
602
603 if (dev->kobj_node) {
604 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
605 sysfs_remove_file(dev->kobj_node, &dev->attr_name);
606 sysfs_remove_file(dev->kobj_node, &dev->attr_props);
607 kobject_del(dev->kobj_node);
608 kobject_put(dev->kobj_node);
609 dev->kobj_node = NULL;
610 }
611 }
612
kfd_build_sysfs_node_entry(struct kfd_topology_device * dev,uint32_t id)613 static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
614 uint32_t id)
615 {
616 struct kfd_iolink_properties *iolink;
617 struct kfd_cache_properties *cache;
618 struct kfd_mem_properties *mem;
619 struct kfd_perf_properties *perf;
620 int ret;
621 uint32_t i, num_attrs;
622 struct attribute **attrs;
623
624 if (WARN_ON(dev->kobj_node))
625 return -EEXIST;
626
627 /*
628 * Creating the sysfs folders
629 */
630 dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
631 if (!dev->kobj_node)
632 return -ENOMEM;
633
634 ret = kobject_init_and_add(dev->kobj_node, &node_type,
635 sys_props.kobj_nodes, "%d", id);
636 if (ret < 0)
637 return ret;
638
639 dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node);
640 if (!dev->kobj_mem)
641 return -ENOMEM;
642
643 dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node);
644 if (!dev->kobj_cache)
645 return -ENOMEM;
646
647 dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node);
648 if (!dev->kobj_iolink)
649 return -ENOMEM;
650
651 dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node);
652 if (!dev->kobj_perf)
653 return -ENOMEM;
654
655 /*
656 * Creating sysfs files for node properties
657 */
658 dev->attr_gpuid.name = "gpu_id";
659 dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE;
660 sysfs_attr_init(&dev->attr_gpuid);
661 dev->attr_name.name = "name";
662 dev->attr_name.mode = KFD_SYSFS_FILE_MODE;
663 sysfs_attr_init(&dev->attr_name);
664 dev->attr_props.name = "properties";
665 dev->attr_props.mode = KFD_SYSFS_FILE_MODE;
666 sysfs_attr_init(&dev->attr_props);
667 ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid);
668 if (ret < 0)
669 return ret;
670 ret = sysfs_create_file(dev->kobj_node, &dev->attr_name);
671 if (ret < 0)
672 return ret;
673 ret = sysfs_create_file(dev->kobj_node, &dev->attr_props);
674 if (ret < 0)
675 return ret;
676
677 i = 0;
678 list_for_each_entry(mem, &dev->mem_props, list) {
679 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
680 if (!mem->kobj)
681 return -ENOMEM;
682 ret = kobject_init_and_add(mem->kobj, &mem_type,
683 dev->kobj_mem, "%d", i);
684 if (ret < 0)
685 return ret;
686
687 mem->attr.name = "properties";
688 mem->attr.mode = KFD_SYSFS_FILE_MODE;
689 sysfs_attr_init(&mem->attr);
690 ret = sysfs_create_file(mem->kobj, &mem->attr);
691 if (ret < 0)
692 return ret;
693 i++;
694 }
695
696 i = 0;
697 list_for_each_entry(cache, &dev->cache_props, list) {
698 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
699 if (!cache->kobj)
700 return -ENOMEM;
701 ret = kobject_init_and_add(cache->kobj, &cache_type,
702 dev->kobj_cache, "%d", i);
703 if (ret < 0)
704 return ret;
705
706 cache->attr.name = "properties";
707 cache->attr.mode = KFD_SYSFS_FILE_MODE;
708 sysfs_attr_init(&cache->attr);
709 ret = sysfs_create_file(cache->kobj, &cache->attr);
710 if (ret < 0)
711 return ret;
712 i++;
713 }
714
715 i = 0;
716 list_for_each_entry(iolink, &dev->io_link_props, list) {
717 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
718 if (!iolink->kobj)
719 return -ENOMEM;
720 ret = kobject_init_and_add(iolink->kobj, &iolink_type,
721 dev->kobj_iolink, "%d", i);
722 if (ret < 0)
723 return ret;
724
725 iolink->attr.name = "properties";
726 iolink->attr.mode = KFD_SYSFS_FILE_MODE;
727 sysfs_attr_init(&iolink->attr);
728 ret = sysfs_create_file(iolink->kobj, &iolink->attr);
729 if (ret < 0)
730 return ret;
731 i++;
732 }
733
734 /* All hardware blocks have the same number of attributes. */
735 num_attrs = ARRAY_SIZE(perf_attr_iommu);
736 list_for_each_entry(perf, &dev->perf_props, list) {
737 perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr)
738 * num_attrs + sizeof(struct attribute_group),
739 GFP_KERNEL);
740 if (!perf->attr_group)
741 return -ENOMEM;
742
743 attrs = (struct attribute **)(perf->attr_group + 1);
744 if (!strcmp(perf->block_name, "iommu")) {
745 /* Information of IOMMU's num_counters and counter_ids is shown
746 * under /sys/bus/event_source/devices/amd_iommu. We don't
747 * duplicate here.
748 */
749 perf_attr_iommu[0].data = perf->max_concurrent;
750 for (i = 0; i < num_attrs; i++)
751 attrs[i] = &perf_attr_iommu[i].attr.attr;
752 }
753 perf->attr_group->name = perf->block_name;
754 perf->attr_group->attrs = attrs;
755 ret = sysfs_create_group(dev->kobj_perf, perf->attr_group);
756 if (ret < 0)
757 return ret;
758 }
759
760 return 0;
761 }
762
763 /* Called with write topology lock acquired */
kfd_build_sysfs_node_tree(void)764 static int kfd_build_sysfs_node_tree(void)
765 {
766 struct kfd_topology_device *dev;
767 int ret;
768 uint32_t i = 0;
769
770 list_for_each_entry(dev, &topology_device_list, list) {
771 ret = kfd_build_sysfs_node_entry(dev, i);
772 if (ret < 0)
773 return ret;
774 i++;
775 }
776
777 return 0;
778 }
779
780 /* Called with write topology lock acquired */
kfd_remove_sysfs_node_tree(void)781 static void kfd_remove_sysfs_node_tree(void)
782 {
783 struct kfd_topology_device *dev;
784
785 list_for_each_entry(dev, &topology_device_list, list)
786 kfd_remove_sysfs_node_entry(dev);
787 }
788
kfd_topology_update_sysfs(void)789 static int kfd_topology_update_sysfs(void)
790 {
791 int ret;
792
793 pr_info("Creating topology SYSFS entries\n");
794 if (!sys_props.kobj_topology) {
795 sys_props.kobj_topology =
796 kfd_alloc_struct(sys_props.kobj_topology);
797 if (!sys_props.kobj_topology)
798 return -ENOMEM;
799
800 ret = kobject_init_and_add(sys_props.kobj_topology,
801 &sysprops_type, &kfd_device->kobj,
802 "topology");
803 if (ret < 0)
804 return ret;
805
806 sys_props.kobj_nodes = kobject_create_and_add("nodes",
807 sys_props.kobj_topology);
808 if (!sys_props.kobj_nodes)
809 return -ENOMEM;
810
811 sys_props.attr_genid.name = "generation_id";
812 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE;
813 sysfs_attr_init(&sys_props.attr_genid);
814 ret = sysfs_create_file(sys_props.kobj_topology,
815 &sys_props.attr_genid);
816 if (ret < 0)
817 return ret;
818
819 sys_props.attr_props.name = "system_properties";
820 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE;
821 sysfs_attr_init(&sys_props.attr_props);
822 ret = sysfs_create_file(sys_props.kobj_topology,
823 &sys_props.attr_props);
824 if (ret < 0)
825 return ret;
826 }
827
828 kfd_remove_sysfs_node_tree();
829
830 return kfd_build_sysfs_node_tree();
831 }
832
kfd_topology_release_sysfs(void)833 static void kfd_topology_release_sysfs(void)
834 {
835 kfd_remove_sysfs_node_tree();
836 if (sys_props.kobj_topology) {
837 sysfs_remove_file(sys_props.kobj_topology,
838 &sys_props.attr_genid);
839 sysfs_remove_file(sys_props.kobj_topology,
840 &sys_props.attr_props);
841 if (sys_props.kobj_nodes) {
842 kobject_del(sys_props.kobj_nodes);
843 kobject_put(sys_props.kobj_nodes);
844 sys_props.kobj_nodes = NULL;
845 }
846 kobject_del(sys_props.kobj_topology);
847 kobject_put(sys_props.kobj_topology);
848 sys_props.kobj_topology = NULL;
849 }
850 }
851
852 /* Called with write topology_lock acquired */
kfd_topology_update_device_list(struct list_head * temp_list,struct list_head * master_list)853 static void kfd_topology_update_device_list(struct list_head *temp_list,
854 struct list_head *master_list)
855 {
856 while (!list_empty(temp_list)) {
857 list_move_tail(temp_list->next, master_list);
858 sys_props.num_devices++;
859 }
860 }
861
kfd_debug_print_topology(void)862 static void kfd_debug_print_topology(void)
863 {
864 struct kfd_topology_device *dev;
865
866 down_read(&topology_lock);
867
868 dev = list_last_entry(&topology_device_list,
869 struct kfd_topology_device, list);
870 if (dev) {
871 if (dev->node_props.cpu_cores_count &&
872 dev->node_props.simd_count) {
873 pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
874 dev->node_props.device_id,
875 dev->node_props.vendor_id);
876 } else if (dev->node_props.cpu_cores_count)
877 pr_info("Topology: Add CPU node\n");
878 else if (dev->node_props.simd_count)
879 pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
880 dev->node_props.device_id,
881 dev->node_props.vendor_id);
882 }
883 up_read(&topology_lock);
884 }
885
886 /* Helper function for intializing platform_xx members of
887 * kfd_system_properties. Uses OEM info from the last CPU/APU node.
888 */
kfd_update_system_properties(void)889 static void kfd_update_system_properties(void)
890 {
891 struct kfd_topology_device *dev;
892
893 down_read(&topology_lock);
894 dev = list_last_entry(&topology_device_list,
895 struct kfd_topology_device, list);
896 if (dev) {
897 sys_props.platform_id =
898 (*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK;
899 sys_props.platform_oem = *((uint64_t *)dev->oem_table_id);
900 sys_props.platform_rev = dev->oem_revision;
901 }
902 up_read(&topology_lock);
903 }
904
find_system_memory(const struct dmi_header * dm,void * private)905 static void find_system_memory(const struct dmi_header *dm,
906 void *private)
907 {
908 struct kfd_mem_properties *mem;
909 u16 mem_width, mem_clock;
910 struct kfd_topology_device *kdev =
911 (struct kfd_topology_device *)private;
912 const u8 *dmi_data = (const u8 *)(dm + 1);
913
914 if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) {
915 mem_width = (u16)(*(const u16 *)(dmi_data + 0x6));
916 mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11));
917 list_for_each_entry(mem, &kdev->mem_props, list) {
918 if (mem_width != 0xFFFF && mem_width != 0)
919 mem->width = mem_width;
920 if (mem_clock != 0)
921 mem->mem_clk_max = mem_clock;
922 }
923 }
924 }
925
926 /*
927 * Performance counters information is not part of CRAT but we would like to
928 * put them in the sysfs under topology directory for Thunk to get the data.
929 * This function is called before updating the sysfs.
930 */
kfd_add_perf_to_topology(struct kfd_topology_device * kdev)931 static int kfd_add_perf_to_topology(struct kfd_topology_device *kdev)
932 {
933 /* These are the only counters supported so far */
934 return kfd_iommu_add_perf_counters(kdev);
935 }
936
937 /* kfd_add_non_crat_information - Add information that is not currently
938 * defined in CRAT but is necessary for KFD topology
939 * @dev - topology device to which addition info is added
940 */
kfd_add_non_crat_information(struct kfd_topology_device * kdev)941 static void kfd_add_non_crat_information(struct kfd_topology_device *kdev)
942 {
943 /* Check if CPU only node. */
944 if (!kdev->gpu) {
945 /* Add system memory information */
946 dmi_walk(find_system_memory, kdev);
947 }
948 /* TODO: For GPU node, rearrange code from kfd_topology_add_device */
949 }
950
951 /* kfd_is_acpi_crat_invalid - CRAT from ACPI is valid only for AMD APU devices.
952 * Ignore CRAT for all other devices. AMD APU is identified if both CPU
953 * and GPU cores are present.
954 * @device_list - topology device list created by parsing ACPI CRAT table.
955 * @return - TRUE if invalid, FALSE is valid.
956 */
kfd_is_acpi_crat_invalid(struct list_head * device_list)957 static bool kfd_is_acpi_crat_invalid(struct list_head *device_list)
958 {
959 struct kfd_topology_device *dev;
960
961 list_for_each_entry(dev, device_list, list) {
962 if (dev->node_props.cpu_cores_count &&
963 dev->node_props.simd_count)
964 return false;
965 }
966 pr_info("Ignoring ACPI CRAT on non-APU system\n");
967 return true;
968 }
969
kfd_topology_init(void)970 int kfd_topology_init(void)
971 {
972 void *crat_image = NULL;
973 size_t image_size = 0;
974 int ret;
975 struct list_head temp_topology_device_list;
976 int cpu_only_node = 0;
977 struct kfd_topology_device *kdev;
978 int proximity_domain;
979
980 /* topology_device_list - Master list of all topology devices
981 * temp_topology_device_list - temporary list created while parsing CRAT
982 * or VCRAT. Once parsing is complete the contents of list is moved to
983 * topology_device_list
984 */
985
986 /* Initialize the head for the both the lists */
987 INIT_LIST_HEAD(&topology_device_list);
988 INIT_LIST_HEAD(&temp_topology_device_list);
989 init_rwsem(&topology_lock);
990
991 memset(&sys_props, 0, sizeof(sys_props));
992
993 /* Proximity domains in ACPI CRAT tables start counting at
994 * 0. The same should be true for virtual CRAT tables created
995 * at this stage. GPUs added later in kfd_topology_add_device
996 * use a counter.
997 */
998 proximity_domain = 0;
999
1000 /*
1001 * Get the CRAT image from the ACPI. If ACPI doesn't have one
1002 * or if ACPI CRAT is invalid create a virtual CRAT.
1003 * NOTE: The current implementation expects all AMD APUs to have
1004 * CRAT. If no CRAT is available, it is assumed to be a CPU
1005 */
1006 ret = kfd_create_crat_image_acpi(&crat_image, &image_size);
1007 if (!ret) {
1008 ret = kfd_parse_crat_table(crat_image,
1009 &temp_topology_device_list,
1010 proximity_domain);
1011 if (ret ||
1012 kfd_is_acpi_crat_invalid(&temp_topology_device_list)) {
1013 kfd_release_topology_device_list(
1014 &temp_topology_device_list);
1015 kfd_destroy_crat_image(crat_image);
1016 crat_image = NULL;
1017 }
1018 }
1019
1020 if (!crat_image) {
1021 ret = kfd_create_crat_image_virtual(&crat_image, &image_size,
1022 COMPUTE_UNIT_CPU, NULL,
1023 proximity_domain);
1024 cpu_only_node = 1;
1025 if (ret) {
1026 pr_err("Error creating VCRAT table for CPU\n");
1027 return ret;
1028 }
1029
1030 ret = kfd_parse_crat_table(crat_image,
1031 &temp_topology_device_list,
1032 proximity_domain);
1033 if (ret) {
1034 pr_err("Error parsing VCRAT table for CPU\n");
1035 goto err;
1036 }
1037 }
1038
1039 kdev = list_first_entry(&temp_topology_device_list,
1040 struct kfd_topology_device, list);
1041 kfd_add_perf_to_topology(kdev);
1042
1043 down_write(&topology_lock);
1044 kfd_topology_update_device_list(&temp_topology_device_list,
1045 &topology_device_list);
1046 atomic_set(&topology_crat_proximity_domain, sys_props.num_devices-1);
1047 ret = kfd_topology_update_sysfs();
1048 up_write(&topology_lock);
1049
1050 if (!ret) {
1051 sys_props.generation_count++;
1052 kfd_update_system_properties();
1053 kfd_debug_print_topology();
1054 pr_info("Finished initializing topology\n");
1055 } else
1056 pr_err("Failed to update topology in sysfs ret=%d\n", ret);
1057
1058 /* For nodes with GPU, this information gets added
1059 * when GPU is detected (kfd_topology_add_device).
1060 */
1061 if (cpu_only_node) {
1062 /* Add additional information to CPU only node created above */
1063 down_write(&topology_lock);
1064 kdev = list_first_entry(&topology_device_list,
1065 struct kfd_topology_device, list);
1066 up_write(&topology_lock);
1067 kfd_add_non_crat_information(kdev);
1068 }
1069
1070 err:
1071 kfd_destroy_crat_image(crat_image);
1072 return ret;
1073 }
1074
kfd_topology_shutdown(void)1075 void kfd_topology_shutdown(void)
1076 {
1077 down_write(&topology_lock);
1078 kfd_topology_release_sysfs();
1079 kfd_release_live_view();
1080 up_write(&topology_lock);
1081 }
1082
kfd_generate_gpu_id(struct kfd_dev * gpu)1083 static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
1084 {
1085 uint32_t hashout;
1086 uint32_t buf[7];
1087 uint64_t local_mem_size;
1088 int i;
1089 struct kfd_local_mem_info local_mem_info;
1090
1091 if (!gpu)
1092 return 0;
1093
1094 amdgpu_amdkfd_get_local_mem_info(gpu->kgd, &local_mem_info);
1095
1096 local_mem_size = local_mem_info.local_mem_size_private +
1097 local_mem_info.local_mem_size_public;
1098
1099 buf[0] = gpu->pdev->devfn;
1100 buf[1] = gpu->pdev->subsystem_vendor |
1101 (gpu->pdev->subsystem_device << 16);
1102 buf[2] = pci_domain_nr(gpu->pdev->bus);
1103 buf[3] = gpu->pdev->device;
1104 buf[4] = gpu->pdev->bus->number;
1105 buf[5] = lower_32_bits(local_mem_size);
1106 buf[6] = upper_32_bits(local_mem_size);
1107
1108 for (i = 0, hashout = 0; i < 7; i++)
1109 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH);
1110
1111 return hashout;
1112 }
1113 /* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
1114 * the GPU device is not already present in the topology device
1115 * list then return NULL. This means a new topology device has to
1116 * be created for this GPU.
1117 */
kfd_assign_gpu(struct kfd_dev * gpu)1118 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
1119 {
1120 struct kfd_topology_device *dev;
1121 struct kfd_topology_device *out_dev = NULL;
1122 struct kfd_mem_properties *mem;
1123 struct kfd_cache_properties *cache;
1124 struct kfd_iolink_properties *iolink;
1125
1126 down_write(&topology_lock);
1127 list_for_each_entry(dev, &topology_device_list, list) {
1128 /* Discrete GPUs need their own topology device list
1129 * entries. Don't assign them to CPU/APU nodes.
1130 */
1131 if (!gpu->device_info->needs_iommu_device &&
1132 dev->node_props.cpu_cores_count)
1133 continue;
1134
1135 if (!dev->gpu && (dev->node_props.simd_count > 0)) {
1136 dev->gpu = gpu;
1137 out_dev = dev;
1138
1139 list_for_each_entry(mem, &dev->mem_props, list)
1140 mem->gpu = dev->gpu;
1141 list_for_each_entry(cache, &dev->cache_props, list)
1142 cache->gpu = dev->gpu;
1143 list_for_each_entry(iolink, &dev->io_link_props, list)
1144 iolink->gpu = dev->gpu;
1145 break;
1146 }
1147 }
1148 up_write(&topology_lock);
1149 return out_dev;
1150 }
1151
kfd_notify_gpu_change(uint32_t gpu_id,int arrival)1152 static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
1153 {
1154 /*
1155 * TODO: Generate an event for thunk about the arrival/removal
1156 * of the GPU
1157 */
1158 }
1159
1160 /* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
1161 * patch this after CRAT parsing.
1162 */
kfd_fill_mem_clk_max_info(struct kfd_topology_device * dev)1163 static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
1164 {
1165 struct kfd_mem_properties *mem;
1166 struct kfd_local_mem_info local_mem_info;
1167
1168 if (!dev)
1169 return;
1170
1171 /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
1172 * single bank of VRAM local memory.
1173 * for dGPUs - VCRAT reports only one bank of Local Memory
1174 * for APUs - If CRAT from ACPI reports more than one bank, then
1175 * all the banks will report the same mem_clk_max information
1176 */
1177 amdgpu_amdkfd_get_local_mem_info(dev->gpu->kgd, &local_mem_info);
1178
1179 list_for_each_entry(mem, &dev->mem_props, list)
1180 mem->mem_clk_max = local_mem_info.mem_clk_max;
1181 }
1182
kfd_fill_iolink_non_crat_info(struct kfd_topology_device * dev)1183 static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
1184 {
1185 struct kfd_iolink_properties *link, *cpu_link;
1186 struct kfd_topology_device *cpu_dev;
1187 uint32_t cap;
1188 uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
1189 uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
1190
1191 if (!dev || !dev->gpu)
1192 return;
1193
1194 pcie_capability_read_dword(dev->gpu->pdev,
1195 PCI_EXP_DEVCAP2, &cap);
1196
1197 if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
1198 PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
1199 cpu_flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1200 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1201
1202 if (!dev->gpu->pci_atomic_requested ||
1203 dev->gpu->device_info->asic_family == CHIP_HAWAII)
1204 flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1205 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1206
1207 /* GPU only creates direct links so apply flags setting to all */
1208 list_for_each_entry(link, &dev->io_link_props, list) {
1209 link->flags = flag;
1210 cpu_dev = kfd_topology_device_by_proximity_domain(
1211 link->node_to);
1212 if (cpu_dev) {
1213 list_for_each_entry(cpu_link,
1214 &cpu_dev->io_link_props, list)
1215 if (cpu_link->node_to == link->node_from)
1216 cpu_link->flags = cpu_flag;
1217 }
1218 }
1219 }
1220
kfd_topology_add_device(struct kfd_dev * gpu)1221 int kfd_topology_add_device(struct kfd_dev *gpu)
1222 {
1223 uint32_t gpu_id;
1224 struct kfd_topology_device *dev;
1225 struct kfd_cu_info cu_info;
1226 int res = 0;
1227 struct list_head temp_topology_device_list;
1228 void *crat_image = NULL;
1229 size_t image_size = 0;
1230 int proximity_domain;
1231 struct amdgpu_ras *ctx;
1232
1233 INIT_LIST_HEAD(&temp_topology_device_list);
1234
1235 gpu_id = kfd_generate_gpu_id(gpu);
1236
1237 pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
1238
1239 proximity_domain = atomic_inc_return(&topology_crat_proximity_domain);
1240
1241 /* Check to see if this gpu device exists in the topology_device_list.
1242 * If so, assign the gpu to that device,
1243 * else create a Virtual CRAT for this gpu device and then parse that
1244 * CRAT to create a new topology device. Once created assign the gpu to
1245 * that topology device
1246 */
1247 dev = kfd_assign_gpu(gpu);
1248 if (!dev) {
1249 res = kfd_create_crat_image_virtual(&crat_image, &image_size,
1250 COMPUTE_UNIT_GPU, gpu,
1251 proximity_domain);
1252 if (res) {
1253 pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
1254 gpu_id);
1255 return res;
1256 }
1257 res = kfd_parse_crat_table(crat_image,
1258 &temp_topology_device_list,
1259 proximity_domain);
1260 if (res) {
1261 pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
1262 gpu_id);
1263 goto err;
1264 }
1265
1266 down_write(&topology_lock);
1267 kfd_topology_update_device_list(&temp_topology_device_list,
1268 &topology_device_list);
1269
1270 /* Update the SYSFS tree, since we added another topology
1271 * device
1272 */
1273 res = kfd_topology_update_sysfs();
1274 up_write(&topology_lock);
1275
1276 if (!res)
1277 sys_props.generation_count++;
1278 else
1279 pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
1280 gpu_id, res);
1281 dev = kfd_assign_gpu(gpu);
1282 if (WARN_ON(!dev)) {
1283 res = -ENODEV;
1284 goto err;
1285 }
1286 }
1287
1288 dev->gpu_id = gpu_id;
1289 gpu->id = gpu_id;
1290
1291 /* TODO: Move the following lines to function
1292 * kfd_add_non_crat_information
1293 */
1294
1295 /* Fill-in additional information that is not available in CRAT but
1296 * needed for the topology
1297 */
1298
1299 amdgpu_amdkfd_get_cu_info(dev->gpu->kgd, &cu_info);
1300
1301 strncpy(dev->node_props.name, gpu->device_info->asic_name,
1302 KFD_TOPOLOGY_PUBLIC_NAME_SIZE);
1303
1304 dev->node_props.simd_arrays_per_engine =
1305 cu_info.num_shader_arrays_per_engine;
1306
1307 dev->node_props.vendor_id = gpu->pdev->vendor;
1308 dev->node_props.device_id = gpu->pdev->device;
1309 dev->node_props.location_id = pci_dev_id(gpu->pdev);
1310 dev->node_props.max_engine_clk_fcompute =
1311 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->kgd);
1312 dev->node_props.max_engine_clk_ccompute =
1313 cpufreq_quick_get_max(0) / 1000;
1314 dev->node_props.drm_render_minor =
1315 gpu->shared_resources.drm_render_minor;
1316
1317 dev->node_props.hive_id = gpu->hive_id;
1318 dev->node_props.num_sdma_engines = gpu->device_info->num_sdma_engines;
1319 dev->node_props.num_sdma_xgmi_engines =
1320 gpu->device_info->num_xgmi_sdma_engines;
1321 dev->node_props.num_sdma_queues_per_engine =
1322 gpu->device_info->num_sdma_queues_per_engine;
1323 dev->node_props.num_gws = (hws_gws_support &&
1324 dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
1325 amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0;
1326 dev->node_props.num_cp_queues = get_queues_num(dev->gpu->dqm);
1327
1328 kfd_fill_mem_clk_max_info(dev);
1329 kfd_fill_iolink_non_crat_info(dev);
1330
1331 switch (dev->gpu->device_info->asic_family) {
1332 case CHIP_KAVERI:
1333 case CHIP_HAWAII:
1334 case CHIP_TONGA:
1335 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
1336 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1337 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1338 break;
1339 case CHIP_CARRIZO:
1340 case CHIP_FIJI:
1341 case CHIP_POLARIS10:
1342 case CHIP_POLARIS11:
1343 case CHIP_POLARIS12:
1344 case CHIP_VEGAM:
1345 pr_debug("Adding doorbell packet type capability\n");
1346 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
1347 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1348 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1349 break;
1350 case CHIP_VEGA10:
1351 case CHIP_VEGA12:
1352 case CHIP_VEGA20:
1353 case CHIP_RAVEN:
1354 case CHIP_RENOIR:
1355 case CHIP_ARCTURUS:
1356 case CHIP_NAVI10:
1357 case CHIP_NAVI12:
1358 case CHIP_NAVI14:
1359 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
1360 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1361 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1362 break;
1363 default:
1364 WARN(1, "Unexpected ASIC family %u",
1365 dev->gpu->device_info->asic_family);
1366 }
1367
1368 /*
1369 * Overwrite ATS capability according to needs_iommu_device to fix
1370 * potential missing corresponding bit in CRAT of BIOS.
1371 */
1372 if (dev->gpu->device_info->needs_iommu_device)
1373 dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
1374 else
1375 dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
1376
1377 /* Fix errors in CZ CRAT.
1378 * simd_count: Carrizo CRAT reports wrong simd_count, probably
1379 * because it doesn't consider masked out CUs
1380 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
1381 */
1382 if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
1383 dev->node_props.simd_count =
1384 cu_info.simd_per_cu * cu_info.cu_active_number;
1385 dev->node_props.max_waves_per_simd = 10;
1386 }
1387
1388 ctx = amdgpu_ras_get_context((struct amdgpu_device *)(dev->gpu->kgd));
1389 if (ctx) {
1390 /* kfd only concerns sram ecc on GFX/SDMA and HBM ecc on UMC */
1391 dev->node_props.capability |=
1392 (((ctx->features & BIT(AMDGPU_RAS_BLOCK__SDMA)) != 0) ||
1393 ((ctx->features & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0)) ?
1394 HSA_CAP_SRAM_EDCSUPPORTED : 0;
1395 dev->node_props.capability |= ((ctx->features & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
1396 HSA_CAP_MEM_EDCSUPPORTED : 0;
1397
1398 dev->node_props.capability |= (ctx->features != 0) ?
1399 HSA_CAP_RASEVENTNOTIFY : 0;
1400 }
1401
1402 kfd_debug_print_topology();
1403
1404 if (!res)
1405 kfd_notify_gpu_change(gpu_id, 1);
1406 err:
1407 kfd_destroy_crat_image(crat_image);
1408 return res;
1409 }
1410
kfd_topology_remove_device(struct kfd_dev * gpu)1411 int kfd_topology_remove_device(struct kfd_dev *gpu)
1412 {
1413 struct kfd_topology_device *dev, *tmp;
1414 uint32_t gpu_id;
1415 int res = -ENODEV;
1416
1417 down_write(&topology_lock);
1418
1419 list_for_each_entry_safe(dev, tmp, &topology_device_list, list)
1420 if (dev->gpu == gpu) {
1421 gpu_id = dev->gpu_id;
1422 kfd_remove_sysfs_node_entry(dev);
1423 kfd_release_topology_device(dev);
1424 sys_props.num_devices--;
1425 res = 0;
1426 if (kfd_topology_update_sysfs() < 0)
1427 kfd_topology_release_sysfs();
1428 break;
1429 }
1430
1431 up_write(&topology_lock);
1432
1433 if (!res)
1434 kfd_notify_gpu_change(gpu_id, 0);
1435
1436 return res;
1437 }
1438
1439 /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
1440 * topology. If GPU device is found @idx, then valid kfd_dev pointer is
1441 * returned through @kdev
1442 * Return - 0: On success (@kdev will be NULL for non GPU nodes)
1443 * -1: If end of list
1444 */
kfd_topology_enum_kfd_devices(uint8_t idx,struct kfd_dev ** kdev)1445 int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
1446 {
1447
1448 struct kfd_topology_device *top_dev;
1449 uint8_t device_idx = 0;
1450
1451 *kdev = NULL;
1452 down_read(&topology_lock);
1453
1454 list_for_each_entry(top_dev, &topology_device_list, list) {
1455 if (device_idx == idx) {
1456 *kdev = top_dev->gpu;
1457 up_read(&topology_lock);
1458 return 0;
1459 }
1460
1461 device_idx++;
1462 }
1463
1464 up_read(&topology_lock);
1465
1466 return -1;
1467
1468 }
1469
kfd_cpumask_to_apic_id(const struct cpumask * cpumask)1470 static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask)
1471 {
1472 int first_cpu_of_numa_node;
1473
1474 if (!cpumask || cpumask == cpu_none_mask)
1475 return -1;
1476 first_cpu_of_numa_node = cpumask_first(cpumask);
1477 if (first_cpu_of_numa_node >= nr_cpu_ids)
1478 return -1;
1479 #ifdef CONFIG_X86_64
1480 return cpu_data(first_cpu_of_numa_node).apicid;
1481 #else
1482 return first_cpu_of_numa_node;
1483 #endif
1484 }
1485
1486 /* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
1487 * of the given NUMA node (numa_node_id)
1488 * Return -1 on failure
1489 */
kfd_numa_node_to_apic_id(int numa_node_id)1490 int kfd_numa_node_to_apic_id(int numa_node_id)
1491 {
1492 if (numa_node_id == -1) {
1493 pr_warn("Invalid NUMA Node. Use online CPU mask\n");
1494 return kfd_cpumask_to_apic_id(cpu_online_mask);
1495 }
1496 return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id));
1497 }
1498
1499 #if defined(CONFIG_DEBUG_FS)
1500
kfd_debugfs_hqds_by_device(struct seq_file * m,void * data)1501 int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data)
1502 {
1503 struct kfd_topology_device *dev;
1504 unsigned int i = 0;
1505 int r = 0;
1506
1507 down_read(&topology_lock);
1508
1509 list_for_each_entry(dev, &topology_device_list, list) {
1510 if (!dev->gpu) {
1511 i++;
1512 continue;
1513 }
1514
1515 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
1516 r = dqm_debugfs_hqds(m, dev->gpu->dqm);
1517 if (r)
1518 break;
1519 }
1520
1521 up_read(&topology_lock);
1522
1523 return r;
1524 }
1525
kfd_debugfs_rls_by_device(struct seq_file * m,void * data)1526 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data)
1527 {
1528 struct kfd_topology_device *dev;
1529 unsigned int i = 0;
1530 int r = 0;
1531
1532 down_read(&topology_lock);
1533
1534 list_for_each_entry(dev, &topology_device_list, list) {
1535 if (!dev->gpu) {
1536 i++;
1537 continue;
1538 }
1539
1540 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
1541 r = pm_debugfs_runlist(m, &dev->gpu->dqm->packets);
1542 if (r)
1543 break;
1544 }
1545
1546 up_read(&topology_lock);
1547
1548 return r;
1549 }
1550
1551 #endif
1552