xref: /netbsd-src/sys/external/bsd/drm2/dist/drm/amd/amdkfd/kfd_crat.c (revision 41ec02673d281bbb3d38e6c78504ce6e30c228c1)
1 /*	$NetBSD: kfd_crat.c,v 1.2 2021/12/18 23:44:59 riastradh Exp $	*/
2 
3 /*
4  * Copyright 2015-2017 Advanced Micro Devices, Inc.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  */
24 
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(0, "$NetBSD: kfd_crat.c,v 1.2 2021/12/18 23:44:59 riastradh Exp $");
27 
28 #include <linux/pci.h>
29 #include <linux/acpi.h>
30 #include "kfd_crat.h"
31 #include "kfd_priv.h"
32 #include "kfd_topology.h"
33 #include "kfd_iommu.h"
34 #include "amdgpu_amdkfd.h"
35 
36 /* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
37  * GPU processor ID are expressed with Bit[31]=1.
38  * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
39  * used in the CRAT.
40  */
41 static uint32_t gpu_processor_id_low = 0x80001000;
42 
43 /* Return the next available gpu_processor_id and increment it for next GPU
44  *	@total_cu_count - Total CUs present in the GPU including ones
45  *			  masked off
46  */
get_and_inc_gpu_processor_id(unsigned int total_cu_count)47 static inline unsigned int get_and_inc_gpu_processor_id(
48 				unsigned int total_cu_count)
49 {
50 	int current_id = gpu_processor_id_low;
51 
52 	gpu_processor_id_low += total_cu_count;
53 	return current_id;
54 }
55 
56 /* Static table to describe GPU Cache information */
57 struct kfd_gpu_cache_info {
58 	uint32_t	cache_size;
59 	uint32_t	cache_level;
60 	uint32_t	flags;
61 	/* Indicates how many Compute Units share this cache
62 	 * Value = 1 indicates the cache is not shared
63 	 */
64 	uint32_t	num_cu_shared;
65 };
66 
67 static struct kfd_gpu_cache_info kaveri_cache_info[] = {
68 	{
69 		/* TCP L1 Cache per CU */
70 		.cache_size = 16,
71 		.cache_level = 1,
72 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
73 				CRAT_CACHE_FLAGS_DATA_CACHE |
74 				CRAT_CACHE_FLAGS_SIMD_CACHE),
75 		.num_cu_shared = 1,
76 
77 	},
78 	{
79 		/* Scalar L1 Instruction Cache (in SQC module) per bank */
80 		.cache_size = 16,
81 		.cache_level = 1,
82 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
83 				CRAT_CACHE_FLAGS_INST_CACHE |
84 				CRAT_CACHE_FLAGS_SIMD_CACHE),
85 		.num_cu_shared = 2,
86 	},
87 	{
88 		/* Scalar L1 Data Cache (in SQC module) per bank */
89 		.cache_size = 8,
90 		.cache_level = 1,
91 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
92 				CRAT_CACHE_FLAGS_DATA_CACHE |
93 				CRAT_CACHE_FLAGS_SIMD_CACHE),
94 		.num_cu_shared = 2,
95 	},
96 
97 	/* TODO: Add L2 Cache information */
98 };
99 
100 
101 static struct kfd_gpu_cache_info carrizo_cache_info[] = {
102 	{
103 		/* TCP L1 Cache per CU */
104 		.cache_size = 16,
105 		.cache_level = 1,
106 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
107 				CRAT_CACHE_FLAGS_DATA_CACHE |
108 				CRAT_CACHE_FLAGS_SIMD_CACHE),
109 		.num_cu_shared = 1,
110 	},
111 	{
112 		/* Scalar L1 Instruction Cache (in SQC module) per bank */
113 		.cache_size = 8,
114 		.cache_level = 1,
115 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
116 				CRAT_CACHE_FLAGS_INST_CACHE |
117 				CRAT_CACHE_FLAGS_SIMD_CACHE),
118 		.num_cu_shared = 4,
119 	},
120 	{
121 		/* Scalar L1 Data Cache (in SQC module) per bank. */
122 		.cache_size = 4,
123 		.cache_level = 1,
124 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
125 				CRAT_CACHE_FLAGS_DATA_CACHE |
126 				CRAT_CACHE_FLAGS_SIMD_CACHE),
127 		.num_cu_shared = 4,
128 	},
129 
130 	/* TODO: Add L2 Cache information */
131 };
132 
133 /* NOTE: In future if more information is added to struct kfd_gpu_cache_info
134  * the following ASICs may need a separate table.
135  */
136 #define hawaii_cache_info kaveri_cache_info
137 #define tonga_cache_info carrizo_cache_info
138 #define fiji_cache_info  carrizo_cache_info
139 #define polaris10_cache_info carrizo_cache_info
140 #define polaris11_cache_info carrizo_cache_info
141 #define polaris12_cache_info carrizo_cache_info
142 #define vegam_cache_info carrizo_cache_info
143 /* TODO - check & update Vega10 cache details */
144 #define vega10_cache_info carrizo_cache_info
145 #define raven_cache_info carrizo_cache_info
146 #define renoir_cache_info carrizo_cache_info
147 /* TODO - check & update Navi10 cache details */
148 #define navi10_cache_info carrizo_cache_info
149 
kfd_populated_cu_info_cpu(struct kfd_topology_device * dev,struct crat_subtype_computeunit * cu)150 static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
151 		struct crat_subtype_computeunit *cu)
152 {
153 	dev->node_props.cpu_cores_count = cu->num_cpu_cores;
154 	dev->node_props.cpu_core_id_base = cu->processor_id_low;
155 	if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
156 		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
157 
158 	pr_debug("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
159 			cu->processor_id_low);
160 }
161 
kfd_populated_cu_info_gpu(struct kfd_topology_device * dev,struct crat_subtype_computeunit * cu)162 static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
163 		struct crat_subtype_computeunit *cu)
164 {
165 	dev->node_props.simd_id_base = cu->processor_id_low;
166 	dev->node_props.simd_count = cu->num_simd_cores;
167 	dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
168 	dev->node_props.max_waves_per_simd = cu->max_waves_simd;
169 	dev->node_props.wave_front_size = cu->wave_front_size;
170 	dev->node_props.array_count = cu->array_count;
171 	dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
172 	dev->node_props.simd_per_cu = cu->num_simd_per_cu;
173 	dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
174 	if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
175 		dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
176 	pr_debug("CU GPU: id_base=%d\n", cu->processor_id_low);
177 }
178 
179 /* kfd_parse_subtype_cu - parse compute unit subtypes and attach it to correct
180  * topology device present in the device_list
181  */
kfd_parse_subtype_cu(struct crat_subtype_computeunit * cu,struct list_head * device_list)182 static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
183 				struct list_head *device_list)
184 {
185 	struct kfd_topology_device *dev;
186 
187 	pr_debug("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
188 			cu->proximity_domain, cu->hsa_capability);
189 	list_for_each_entry(dev, device_list, list) {
190 		if (cu->proximity_domain == dev->proximity_domain) {
191 			if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
192 				kfd_populated_cu_info_cpu(dev, cu);
193 
194 			if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
195 				kfd_populated_cu_info_gpu(dev, cu);
196 			break;
197 		}
198 	}
199 
200 	return 0;
201 }
202 
203 static struct kfd_mem_properties *
find_subtype_mem(uint32_t heap_type,uint32_t flags,uint32_t width,struct kfd_topology_device * dev)204 find_subtype_mem(uint32_t heap_type, uint32_t flags, uint32_t width,
205 		struct kfd_topology_device *dev)
206 {
207 	struct kfd_mem_properties *props;
208 
209 	list_for_each_entry(props, &dev->mem_props, list) {
210 		if (props->heap_type == heap_type
211 				&& props->flags == flags
212 				&& props->width == width)
213 			return props;
214 	}
215 
216 	return NULL;
217 }
218 /* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
219  * topology device present in the device_list
220  */
kfd_parse_subtype_mem(struct crat_subtype_memory * mem,struct list_head * device_list)221 static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
222 				struct list_head *device_list)
223 {
224 	struct kfd_mem_properties *props;
225 	struct kfd_topology_device *dev;
226 	uint32_t heap_type;
227 	uint64_t size_in_bytes;
228 	uint32_t flags = 0;
229 	uint32_t width;
230 
231 	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
232 			mem->proximity_domain);
233 	list_for_each_entry(dev, device_list, list) {
234 		if (mem->proximity_domain == dev->proximity_domain) {
235 			/* We're on GPU node */
236 			if (dev->node_props.cpu_cores_count == 0) {
237 				/* APU */
238 				if (mem->visibility_type == 0)
239 					heap_type =
240 						HSA_MEM_HEAP_TYPE_FB_PRIVATE;
241 				/* dGPU */
242 				else
243 					heap_type = mem->visibility_type;
244 			} else
245 				heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
246 
247 			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
248 				flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
249 			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
250 				flags |= HSA_MEM_FLAGS_NON_VOLATILE;
251 
252 			size_in_bytes =
253 				((uint64_t)mem->length_high << 32) +
254 							mem->length_low;
255 			width = mem->width;
256 
257 			/* Multiple banks of the same type are aggregated into
258 			 * one. User mode doesn't care about multiple physical
259 			 * memory segments. It's managed as a single virtual
260 			 * heap for user mode.
261 			 */
262 			props = find_subtype_mem(heap_type, flags, width, dev);
263 			if (props) {
264 				props->size_in_bytes += size_in_bytes;
265 				break;
266 			}
267 
268 			props = kfd_alloc_struct(props);
269 			if (!props)
270 				return -ENOMEM;
271 
272 			props->heap_type = heap_type;
273 			props->flags = flags;
274 			props->size_in_bytes = size_in_bytes;
275 			props->width = width;
276 
277 			dev->node_props.mem_banks_count++;
278 			list_add_tail(&props->list, &dev->mem_props);
279 
280 			break;
281 		}
282 	}
283 
284 	return 0;
285 }
286 
287 /* kfd_parse_subtype_cache - parse cache subtypes and attach it to correct
288  * topology device present in the device_list
289  */
kfd_parse_subtype_cache(struct crat_subtype_cache * cache,struct list_head * device_list)290 static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
291 			struct list_head *device_list)
292 {
293 	struct kfd_cache_properties *props;
294 	struct kfd_topology_device *dev;
295 	uint32_t id;
296 	uint32_t total_num_of_cu;
297 
298 	id = cache->processor_id_low;
299 
300 	pr_debug("Found cache entry in CRAT table with processor_id=%d\n", id);
301 	list_for_each_entry(dev, device_list, list) {
302 		total_num_of_cu = (dev->node_props.array_count *
303 					dev->node_props.cu_per_simd_array);
304 
305 		/* Cache infomration in CRAT doesn't have proximity_domain
306 		 * information as it is associated with a CPU core or GPU
307 		 * Compute Unit. So map the cache using CPU core Id or SIMD
308 		 * (GPU) ID.
309 		 * TODO: This works because currently we can safely assume that
310 		 *  Compute Units are parsed before caches are parsed. In
311 		 *  future, remove this dependency
312 		 */
313 		if ((id >= dev->node_props.cpu_core_id_base &&
314 			id <= dev->node_props.cpu_core_id_base +
315 				dev->node_props.cpu_cores_count) ||
316 			(id >= dev->node_props.simd_id_base &&
317 			id < dev->node_props.simd_id_base +
318 				total_num_of_cu)) {
319 			props = kfd_alloc_struct(props);
320 			if (!props)
321 				return -ENOMEM;
322 
323 			props->processor_id_low = id;
324 			props->cache_level = cache->cache_level;
325 			props->cache_size = cache->cache_size;
326 			props->cacheline_size = cache->cache_line_size;
327 			props->cachelines_per_tag = cache->lines_per_tag;
328 			props->cache_assoc = cache->associativity;
329 			props->cache_latency = cache->cache_latency;
330 			memcpy(props->sibling_map, cache->sibling_map,
331 					sizeof(props->sibling_map));
332 
333 			if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
334 				props->cache_type |= HSA_CACHE_TYPE_DATA;
335 			if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
336 				props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
337 			if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
338 				props->cache_type |= HSA_CACHE_TYPE_CPU;
339 			if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
340 				props->cache_type |= HSA_CACHE_TYPE_HSACU;
341 
342 			dev->cache_count++;
343 			dev->node_props.caches_count++;
344 			list_add_tail(&props->list, &dev->cache_props);
345 
346 			break;
347 		}
348 	}
349 
350 	return 0;
351 }
352 
353 /* kfd_parse_subtype_iolink - parse iolink subtypes and attach it to correct
354  * topology device present in the device_list
355  */
kfd_parse_subtype_iolink(struct crat_subtype_iolink * iolink,struct list_head * device_list)356 static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
357 					struct list_head *device_list)
358 {
359 	struct kfd_iolink_properties *props = NULL, *props2;
360 	struct kfd_topology_device *dev, *to_dev;
361 	uint32_t id_from;
362 	uint32_t id_to;
363 
364 	id_from = iolink->proximity_domain_from;
365 	id_to = iolink->proximity_domain_to;
366 
367 	pr_debug("Found IO link entry in CRAT table with id_from=%d, id_to %d\n",
368 			id_from, id_to);
369 	list_for_each_entry(dev, device_list, list) {
370 		if (id_from == dev->proximity_domain) {
371 			props = kfd_alloc_struct(props);
372 			if (!props)
373 				return -ENOMEM;
374 
375 			props->node_from = id_from;
376 			props->node_to = id_to;
377 			props->ver_maj = iolink->version_major;
378 			props->ver_min = iolink->version_minor;
379 			props->iolink_type = iolink->io_interface_type;
380 
381 			if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
382 				props->weight = 20;
383 			else if (props->iolink_type == CRAT_IOLINK_TYPE_XGMI)
384 				props->weight = 15 * iolink->num_hops_xgmi;
385 			else
386 				props->weight = node_distance(id_from, id_to);
387 
388 			props->min_latency = iolink->minimum_latency;
389 			props->max_latency = iolink->maximum_latency;
390 			props->min_bandwidth = iolink->minimum_bandwidth_mbs;
391 			props->max_bandwidth = iolink->maximum_bandwidth_mbs;
392 			props->rec_transfer_size =
393 					iolink->recommended_transfer_size;
394 
395 			dev->io_link_count++;
396 			dev->node_props.io_links_count++;
397 			list_add_tail(&props->list, &dev->io_link_props);
398 			break;
399 		}
400 	}
401 
402 	/* CPU topology is created before GPUs are detected, so CPU->GPU
403 	 * links are not built at that time. If a PCIe type is discovered, it
404 	 * means a GPU is detected and we are adding GPU->CPU to the topology.
405 	 * At this time, also add the corresponded CPU->GPU link if GPU
406 	 * is large bar.
407 	 * For xGMI, we only added the link with one direction in the crat
408 	 * table, add corresponded reversed direction link now.
409 	 */
410 	if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
411 		to_dev = kfd_topology_device_by_proximity_domain(id_to);
412 		if (!to_dev)
413 			return -ENODEV;
414 		/* same everything but the other direction */
415 		props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
416 		props2->node_from = id_to;
417 		props2->node_to = id_from;
418 		props2->kobj = NULL;
419 		to_dev->io_link_count++;
420 		to_dev->node_props.io_links_count++;
421 		list_add_tail(&props2->list, &to_dev->io_link_props);
422 	}
423 
424 	return 0;
425 }
426 
427 /* kfd_parse_subtype - parse subtypes and attach it to correct topology device
428  * present in the device_list
429  *	@sub_type_hdr - subtype section of crat_image
430  *	@device_list - list of topology devices present in this crat_image
431  */
kfd_parse_subtype(struct crat_subtype_generic * sub_type_hdr,struct list_head * device_list)432 static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr,
433 				struct list_head *device_list)
434 {
435 	struct crat_subtype_computeunit *cu;
436 	struct crat_subtype_memory *mem;
437 	struct crat_subtype_cache *cache;
438 	struct crat_subtype_iolink *iolink;
439 	int ret = 0;
440 
441 	switch (sub_type_hdr->type) {
442 	case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
443 		cu = (struct crat_subtype_computeunit *)sub_type_hdr;
444 		ret = kfd_parse_subtype_cu(cu, device_list);
445 		break;
446 	case CRAT_SUBTYPE_MEMORY_AFFINITY:
447 		mem = (struct crat_subtype_memory *)sub_type_hdr;
448 		ret = kfd_parse_subtype_mem(mem, device_list);
449 		break;
450 	case CRAT_SUBTYPE_CACHE_AFFINITY:
451 		cache = (struct crat_subtype_cache *)sub_type_hdr;
452 		ret = kfd_parse_subtype_cache(cache, device_list);
453 		break;
454 	case CRAT_SUBTYPE_TLB_AFFINITY:
455 		/*
456 		 * For now, nothing to do here
457 		 */
458 		pr_debug("Found TLB entry in CRAT table (not processing)\n");
459 		break;
460 	case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
461 		/*
462 		 * For now, nothing to do here
463 		 */
464 		pr_debug("Found CCOMPUTE entry in CRAT table (not processing)\n");
465 		break;
466 	case CRAT_SUBTYPE_IOLINK_AFFINITY:
467 		iolink = (struct crat_subtype_iolink *)sub_type_hdr;
468 		ret = kfd_parse_subtype_iolink(iolink, device_list);
469 		break;
470 	default:
471 		pr_warn("Unknown subtype %d in CRAT\n",
472 				sub_type_hdr->type);
473 	}
474 
475 	return ret;
476 }
477 
478 /* kfd_parse_crat_table - parse CRAT table. For each node present in CRAT
479  * create a kfd_topology_device and add in to device_list. Also parse
480  * CRAT subtypes and attach it to appropriate kfd_topology_device
481  *	@crat_image - input image containing CRAT
482  *	@device_list - [OUT] list of kfd_topology_device generated after
483  *		       parsing crat_image
484  *	@proximity_domain - Proximity domain of the first device in the table
485  *
486  *	Return - 0 if successful else -ve value
487  */
kfd_parse_crat_table(void * crat_image,struct list_head * device_list,uint32_t proximity_domain)488 int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
489 			 uint32_t proximity_domain)
490 {
491 	struct kfd_topology_device *top_dev = NULL;
492 	struct crat_subtype_generic *sub_type_hdr;
493 	uint16_t node_id;
494 	int ret = 0;
495 	struct crat_header *crat_table = (struct crat_header *)crat_image;
496 	uint16_t num_nodes;
497 	uint32_t image_len;
498 
499 	if (!crat_image)
500 		return -EINVAL;
501 
502 	if (!list_empty(device_list)) {
503 		pr_warn("Error device list should be empty\n");
504 		return -EINVAL;
505 	}
506 
507 	num_nodes = crat_table->num_domains;
508 	image_len = crat_table->length;
509 
510 	pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
511 
512 	for (node_id = 0; node_id < num_nodes; node_id++) {
513 		top_dev = kfd_create_topology_device(device_list);
514 		if (!top_dev)
515 			break;
516 		top_dev->proximity_domain = proximity_domain++;
517 	}
518 
519 	if (!top_dev) {
520 		ret = -ENOMEM;
521 		goto err;
522 	}
523 
524 	memcpy(top_dev->oem_id, crat_table->oem_id, CRAT_OEMID_LENGTH);
525 	memcpy(top_dev->oem_table_id, crat_table->oem_table_id,
526 			CRAT_OEMTABLEID_LENGTH);
527 	top_dev->oem_revision = crat_table->oem_revision;
528 
529 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
530 	while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
531 			((char *)crat_image) + image_len) {
532 		if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
533 			ret = kfd_parse_subtype(sub_type_hdr, device_list);
534 			if (ret)
535 				break;
536 		}
537 
538 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
539 				sub_type_hdr->length);
540 	}
541 
542 err:
543 	if (ret)
544 		kfd_release_topology_device_list(device_list);
545 
546 	return ret;
547 }
548 
549 /* Helper function. See kfd_fill_gpu_cache_info for parameter description */
fill_in_pcache(struct crat_subtype_cache * pcache,struct kfd_gpu_cache_info * pcache_info,struct kfd_cu_info * cu_info,int mem_available,int cu_bitmask,int cache_type,unsigned int cu_processor_id,int cu_block)550 static int fill_in_pcache(struct crat_subtype_cache *pcache,
551 				struct kfd_gpu_cache_info *pcache_info,
552 				struct kfd_cu_info *cu_info,
553 				int mem_available,
554 				int cu_bitmask,
555 				int cache_type, unsigned int cu_processor_id,
556 				int cu_block)
557 {
558 	unsigned int cu_sibling_map_mask;
559 	int first_active_cu;
560 
561 	/* First check if enough memory is available */
562 	if (sizeof(struct crat_subtype_cache) > mem_available)
563 		return -ENOMEM;
564 
565 	cu_sibling_map_mask = cu_bitmask;
566 	cu_sibling_map_mask >>= cu_block;
567 	cu_sibling_map_mask &=
568 		((1 << pcache_info[cache_type].num_cu_shared) - 1);
569 	first_active_cu = ffs(cu_sibling_map_mask);
570 
571 	/* CU could be inactive. In case of shared cache find the first active
572 	 * CU. and incase of non-shared cache check if the CU is inactive. If
573 	 * inactive active skip it
574 	 */
575 	if (first_active_cu) {
576 		memset(pcache, 0, sizeof(struct crat_subtype_cache));
577 		pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
578 		pcache->length = sizeof(struct crat_subtype_cache);
579 		pcache->flags = pcache_info[cache_type].flags;
580 		pcache->processor_id_low = cu_processor_id
581 					 + (first_active_cu - 1);
582 		pcache->cache_level = pcache_info[cache_type].cache_level;
583 		pcache->cache_size = pcache_info[cache_type].cache_size;
584 
585 		/* Sibling map is w.r.t processor_id_low, so shift out
586 		 * inactive CU
587 		 */
588 		cu_sibling_map_mask =
589 			cu_sibling_map_mask >> (first_active_cu - 1);
590 
591 		pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
592 		pcache->sibling_map[1] =
593 				(uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
594 		pcache->sibling_map[2] =
595 				(uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
596 		pcache->sibling_map[3] =
597 				(uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
598 		return 0;
599 	}
600 	return 1;
601 }
602 
603 /* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
604  * tables
605  *
606  *	@kdev - [IN] GPU device
607  *	@gpu_processor_id - [IN] GPU processor ID to which these caches
608  *			    associate
609  *	@available_size - [IN] Amount of memory available in pcache
610  *	@cu_info - [IN] Compute Unit info obtained from KGD
611  *	@pcache - [OUT] memory into which cache data is to be filled in.
612  *	@size_filled - [OUT] amount of data used up in pcache.
613  *	@num_of_entries - [OUT] number of caches added
614  */
kfd_fill_gpu_cache_info(struct kfd_dev * kdev,int gpu_processor_id,int available_size,struct kfd_cu_info * cu_info,struct crat_subtype_cache * pcache,int * size_filled,int * num_of_entries)615 static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
616 			int gpu_processor_id,
617 			int available_size,
618 			struct kfd_cu_info *cu_info,
619 			struct crat_subtype_cache *pcache,
620 			int *size_filled,
621 			int *num_of_entries)
622 {
623 	struct kfd_gpu_cache_info *pcache_info;
624 	int num_of_cache_types = 0;
625 	int i, j, k;
626 	int ct = 0;
627 	int mem_available = available_size;
628 	unsigned int cu_processor_id;
629 	int ret;
630 
631 	switch (kdev->device_info->asic_family) {
632 	case CHIP_KAVERI:
633 		pcache_info = kaveri_cache_info;
634 		num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
635 		break;
636 	case CHIP_HAWAII:
637 		pcache_info = hawaii_cache_info;
638 		num_of_cache_types = ARRAY_SIZE(hawaii_cache_info);
639 		break;
640 	case CHIP_CARRIZO:
641 		pcache_info = carrizo_cache_info;
642 		num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
643 		break;
644 	case CHIP_TONGA:
645 		pcache_info = tonga_cache_info;
646 		num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
647 		break;
648 	case CHIP_FIJI:
649 		pcache_info = fiji_cache_info;
650 		num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
651 		break;
652 	case CHIP_POLARIS10:
653 		pcache_info = polaris10_cache_info;
654 		num_of_cache_types = ARRAY_SIZE(polaris10_cache_info);
655 		break;
656 	case CHIP_POLARIS11:
657 		pcache_info = polaris11_cache_info;
658 		num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
659 		break;
660 	case CHIP_POLARIS12:
661 		pcache_info = polaris12_cache_info;
662 		num_of_cache_types = ARRAY_SIZE(polaris12_cache_info);
663 		break;
664 	case CHIP_VEGAM:
665 		pcache_info = vegam_cache_info;
666 		num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
667 		break;
668 	case CHIP_VEGA10:
669 	case CHIP_VEGA12:
670 	case CHIP_VEGA20:
671 	case CHIP_ARCTURUS:
672 		pcache_info = vega10_cache_info;
673 		num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
674 		break;
675 	case CHIP_RAVEN:
676 		pcache_info = raven_cache_info;
677 		num_of_cache_types = ARRAY_SIZE(raven_cache_info);
678 		break;
679 	case CHIP_RENOIR:
680 		pcache_info = renoir_cache_info;
681 		num_of_cache_types = ARRAY_SIZE(renoir_cache_info);
682 		break;
683 	case CHIP_NAVI10:
684 	case CHIP_NAVI12:
685 	case CHIP_NAVI14:
686 		pcache_info = navi10_cache_info;
687 		num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
688 		break;
689 	default:
690 		return -EINVAL;
691 	}
692 
693 	*size_filled = 0;
694 	*num_of_entries = 0;
695 
696 	/* For each type of cache listed in the kfd_gpu_cache_info table,
697 	 * go through all available Compute Units.
698 	 * The [i,j,k] loop will
699 	 *		if kfd_gpu_cache_info.num_cu_shared = 1
700 	 *			will parse through all available CU
701 	 *		If (kfd_gpu_cache_info.num_cu_shared != 1)
702 	 *			then it will consider only one CU from
703 	 *			the shared unit
704 	 */
705 
706 	for (ct = 0; ct < num_of_cache_types; ct++) {
707 		cu_processor_id = gpu_processor_id;
708 		for (i = 0; i < cu_info->num_shader_engines; i++) {
709 			for (j = 0; j < cu_info->num_shader_arrays_per_engine;
710 				j++) {
711 				for (k = 0; k < cu_info->num_cu_per_sh;
712 					k += pcache_info[ct].num_cu_shared) {
713 
714 					ret = fill_in_pcache(pcache,
715 						pcache_info,
716 						cu_info,
717 						mem_available,
718 						cu_info->cu_bitmap[i % 4][j + i / 4],
719 						ct,
720 						cu_processor_id,
721 						k);
722 
723 					if (ret < 0)
724 						break;
725 
726 					if (!ret) {
727 						pcache++;
728 						(*num_of_entries)++;
729 						mem_available -=
730 							sizeof(*pcache);
731 						(*size_filled) +=
732 							sizeof(*pcache);
733 					}
734 
735 					/* Move to next CU block */
736 					cu_processor_id +=
737 						pcache_info[ct].num_cu_shared;
738 				}
739 			}
740 		}
741 	}
742 
743 	pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
744 
745 	return 0;
746 }
747 
748 /*
749  * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
750  * copies CRAT from ACPI (if available).
751  * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
752  *
753  *	@crat_image: CRAT read from ACPI. If no CRAT in ACPI then
754  *		     crat_image will be NULL
755  *	@size: [OUT] size of crat_image
756  *
757  *	Return 0 if successful else return error code
758  */
kfd_create_crat_image_acpi(void ** crat_image,size_t * size)759 int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
760 {
761 	struct acpi_table_header *crat_table;
762 	acpi_status status;
763 	void *pcrat_image;
764 
765 	if (!crat_image)
766 		return -EINVAL;
767 
768 	*crat_image = NULL;
769 
770 	/* Fetch the CRAT table from ACPI */
771 	status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
772 	if (status == AE_NOT_FOUND) {
773 		pr_warn("CRAT table not found\n");
774 		return -ENODATA;
775 	} else if (ACPI_FAILURE(status)) {
776 		const char *err = acpi_format_exception(status);
777 
778 		pr_err("CRAT table error: %s\n", err);
779 		return -EINVAL;
780 	}
781 
782 	if (ignore_crat) {
783 		pr_info("CRAT table disabled by module option\n");
784 		return -ENODATA;
785 	}
786 
787 	pcrat_image = kmemdup(crat_table, crat_table->length, GFP_KERNEL);
788 	if (!pcrat_image)
789 		return -ENOMEM;
790 
791 	*crat_image = pcrat_image;
792 	*size = crat_table->length;
793 
794 	return 0;
795 }
796 
797 /* Memory required to create Virtual CRAT.
798  * Since there is no easy way to predict the amount of memory required, the
799  * following amount are allocated for CPU and GPU Virtual CRAT. This is
800  * expected to cover all known conditions. But to be safe additional check
801  * is put in the code to ensure we don't overwrite.
802  */
803 #define VCRAT_SIZE_FOR_CPU	(2 * PAGE_SIZE)
804 #define VCRAT_SIZE_FOR_GPU	(4 * PAGE_SIZE)
805 
806 /* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
807  *
808  *	@numa_node_id: CPU NUMA node id
809  *	@avail_size: Available size in the memory
810  *	@sub_type_hdr: Memory into which compute info will be filled in
811  *
812  *	Return 0 if successful else return -ve value
813  */
kfd_fill_cu_for_cpu(int numa_node_id,int * avail_size,int proximity_domain,struct crat_subtype_computeunit * sub_type_hdr)814 static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
815 				int proximity_domain,
816 				struct crat_subtype_computeunit *sub_type_hdr)
817 {
818 	const struct cpumask *cpumask;
819 
820 	*avail_size -= sizeof(struct crat_subtype_computeunit);
821 	if (*avail_size < 0)
822 		return -ENOMEM;
823 
824 	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
825 
826 	/* Fill in subtype header data */
827 	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
828 	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
829 	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
830 
831 	cpumask = cpumask_of_node(numa_node_id);
832 
833 	/* Fill in CU data */
834 	sub_type_hdr->flags |= CRAT_CU_FLAGS_CPU_PRESENT;
835 	sub_type_hdr->proximity_domain = proximity_domain;
836 	sub_type_hdr->processor_id_low = kfd_numa_node_to_apic_id(numa_node_id);
837 	if (sub_type_hdr->processor_id_low == -1)
838 		return -EINVAL;
839 
840 	sub_type_hdr->num_cpu_cores = cpumask_weight(cpumask);
841 
842 	return 0;
843 }
844 
845 /* kfd_fill_mem_info_for_cpu - Fill in Memory info for the given CPU NUMA node
846  *
847  *	@numa_node_id: CPU NUMA node id
848  *	@avail_size: Available size in the memory
849  *	@sub_type_hdr: Memory into which compute info will be filled in
850  *
851  *	Return 0 if successful else return -ve value
852  */
kfd_fill_mem_info_for_cpu(int numa_node_id,int * avail_size,int proximity_domain,struct crat_subtype_memory * sub_type_hdr)853 static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
854 			int proximity_domain,
855 			struct crat_subtype_memory *sub_type_hdr)
856 {
857 	uint64_t mem_in_bytes = 0;
858 	pg_data_t *pgdat;
859 	int zone_type;
860 
861 	*avail_size -= sizeof(struct crat_subtype_memory);
862 	if (*avail_size < 0)
863 		return -ENOMEM;
864 
865 	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
866 
867 	/* Fill in subtype header data */
868 	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
869 	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
870 	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
871 
872 	/* Fill in Memory Subunit data */
873 
874 	/* Unlike si_meminfo, si_meminfo_node is not exported. So
875 	 * the following lines are duplicated from si_meminfo_node
876 	 * function
877 	 */
878 	pgdat = NODE_DATA(numa_node_id);
879 	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
880 		mem_in_bytes += zone_managed_pages(&pgdat->node_zones[zone_type]);
881 	mem_in_bytes <<= PAGE_SHIFT;
882 
883 	sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
884 	sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
885 	sub_type_hdr->proximity_domain = proximity_domain;
886 
887 	return 0;
888 }
889 
890 #ifdef CONFIG_X86_64
kfd_fill_iolink_info_for_cpu(int numa_node_id,int * avail_size,uint32_t * num_entries,struct crat_subtype_iolink * sub_type_hdr)891 static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size,
892 				uint32_t *num_entries,
893 				struct crat_subtype_iolink *sub_type_hdr)
894 {
895 	int nid;
896 	struct cpuinfo_x86 *c = &cpu_data(0);
897 	uint8_t link_type;
898 
899 	if (c->x86_vendor == X86_VENDOR_AMD)
900 		link_type = CRAT_IOLINK_TYPE_HYPERTRANSPORT;
901 	else
902 		link_type = CRAT_IOLINK_TYPE_QPI_1_1;
903 
904 	*num_entries = 0;
905 
906 	/* Create IO links from this node to other CPU nodes */
907 	for_each_online_node(nid) {
908 		if (nid == numa_node_id) /* node itself */
909 			continue;
910 
911 		*avail_size -= sizeof(struct crat_subtype_iolink);
912 		if (*avail_size < 0)
913 			return -ENOMEM;
914 
915 		memset(sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
916 
917 		/* Fill in subtype header data */
918 		sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
919 		sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
920 		sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
921 
922 		/* Fill in IO link data */
923 		sub_type_hdr->proximity_domain_from = numa_node_id;
924 		sub_type_hdr->proximity_domain_to = nid;
925 		sub_type_hdr->io_interface_type = link_type;
926 
927 		(*num_entries)++;
928 		sub_type_hdr++;
929 	}
930 
931 	return 0;
932 }
933 #endif
934 
935 /* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
936  *
937  *	@pcrat_image: Fill in VCRAT for CPU
938  *	@size:	[IN] allocated size of crat_image.
939  *		[OUT] actual size of data filled in crat_image
940  */
kfd_create_vcrat_image_cpu(void * pcrat_image,size_t * size)941 static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
942 {
943 	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
944 	struct acpi_table_header *acpi_table;
945 	acpi_status status;
946 	struct crat_subtype_generic *sub_type_hdr;
947 	int avail_size = *size;
948 	int numa_node_id;
949 #ifdef CONFIG_X86_64
950 	uint32_t entries = 0;
951 #endif
952 	int ret = 0;
953 
954 	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_CPU)
955 		return -EINVAL;
956 
957 	/* Fill in CRAT Header.
958 	 * Modify length and total_entries as subunits are added.
959 	 */
960 	avail_size -= sizeof(struct crat_header);
961 	if (avail_size < 0)
962 		return -ENOMEM;
963 
964 	memset(crat_table, 0, sizeof(struct crat_header));
965 	memcpy(&crat_table->signature, CRAT_SIGNATURE,
966 			sizeof(crat_table->signature));
967 	crat_table->length = sizeof(struct crat_header);
968 
969 	status = acpi_get_table("DSDT", 0, &acpi_table);
970 	if (status != AE_OK)
971 		pr_warn("DSDT table not found for OEM information\n");
972 	else {
973 		crat_table->oem_revision = acpi_table->revision;
974 		memcpy(crat_table->oem_id, acpi_table->oem_id,
975 				CRAT_OEMID_LENGTH);
976 		memcpy(crat_table->oem_table_id, acpi_table->oem_table_id,
977 				CRAT_OEMTABLEID_LENGTH);
978 	}
979 	crat_table->total_entries = 0;
980 	crat_table->num_domains = 0;
981 
982 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
983 
984 	for_each_online_node(numa_node_id) {
985 		if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
986 			continue;
987 
988 		/* Fill in Subtype: Compute Unit */
989 		ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
990 			crat_table->num_domains,
991 			(struct crat_subtype_computeunit *)sub_type_hdr);
992 		if (ret < 0)
993 			return ret;
994 		crat_table->length += sub_type_hdr->length;
995 		crat_table->total_entries++;
996 
997 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
998 			sub_type_hdr->length);
999 
1000 		/* Fill in Subtype: Memory */
1001 		ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
1002 			crat_table->num_domains,
1003 			(struct crat_subtype_memory *)sub_type_hdr);
1004 		if (ret < 0)
1005 			return ret;
1006 		crat_table->length += sub_type_hdr->length;
1007 		crat_table->total_entries++;
1008 
1009 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1010 			sub_type_hdr->length);
1011 
1012 		/* Fill in Subtype: IO Link */
1013 #ifdef CONFIG_X86_64
1014 		ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
1015 				&entries,
1016 				(struct crat_subtype_iolink *)sub_type_hdr);
1017 		if (ret < 0)
1018 			return ret;
1019 		crat_table->length += (sub_type_hdr->length * entries);
1020 		crat_table->total_entries += entries;
1021 
1022 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1023 				sub_type_hdr->length * entries);
1024 #else
1025 		pr_info("IO link not available for non x86 platforms\n");
1026 #endif
1027 
1028 		crat_table->num_domains++;
1029 	}
1030 
1031 	/* TODO: Add cache Subtype for CPU.
1032 	 * Currently, CPU cache information is available in function
1033 	 * detect_cache_attributes(cpu) defined in the file
1034 	 * ./arch/x86/kernel/cpu/intel_cacheinfo.c. This function is not
1035 	 * exported and to get the same information the code needs to be
1036 	 * duplicated.
1037 	 */
1038 
1039 	*size = crat_table->length;
1040 	pr_info("Virtual CRAT table created for CPU\n");
1041 
1042 	return 0;
1043 }
1044 
kfd_fill_gpu_memory_affinity(int * avail_size,struct kfd_dev * kdev,uint8_t type,uint64_t size,struct crat_subtype_memory * sub_type_hdr,uint32_t proximity_domain,const struct kfd_local_mem_info * local_mem_info)1045 static int kfd_fill_gpu_memory_affinity(int *avail_size,
1046 		struct kfd_dev *kdev, uint8_t type, uint64_t size,
1047 		struct crat_subtype_memory *sub_type_hdr,
1048 		uint32_t proximity_domain,
1049 		const struct kfd_local_mem_info *local_mem_info)
1050 {
1051 	*avail_size -= sizeof(struct crat_subtype_memory);
1052 	if (*avail_size < 0)
1053 		return -ENOMEM;
1054 
1055 	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
1056 	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
1057 	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
1058 	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1059 
1060 	sub_type_hdr->proximity_domain = proximity_domain;
1061 
1062 	pr_debug("Fill gpu memory affinity - type 0x%x size 0x%llx\n",
1063 			type, size);
1064 
1065 	sub_type_hdr->length_low = lower_32_bits(size);
1066 	sub_type_hdr->length_high = upper_32_bits(size);
1067 
1068 	sub_type_hdr->width = local_mem_info->vram_width;
1069 	sub_type_hdr->visibility_type = type;
1070 
1071 	return 0;
1072 }
1073 
1074 /* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
1075  * to its NUMA node
1076  *	@avail_size: Available size in the memory
1077  *	@kdev - [IN] GPU device
1078  *	@sub_type_hdr: Memory into which io link info will be filled in
1079  *	@proximity_domain - proximity domain of the GPU node
1080  *
1081  *	Return 0 if successful else return -ve value
1082  */
kfd_fill_gpu_direct_io_link_to_cpu(int * avail_size,struct kfd_dev * kdev,struct crat_subtype_iolink * sub_type_hdr,uint32_t proximity_domain)1083 static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
1084 			struct kfd_dev *kdev,
1085 			struct crat_subtype_iolink *sub_type_hdr,
1086 			uint32_t proximity_domain)
1087 {
1088 	*avail_size -= sizeof(struct crat_subtype_iolink);
1089 	if (*avail_size < 0)
1090 		return -ENOMEM;
1091 
1092 	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1093 
1094 	/* Fill in subtype header data */
1095 	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1096 	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1097 	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1098 	if (kfd_dev_is_large_bar(kdev))
1099 		sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1100 
1101 	/* Fill in IOLINK subtype.
1102 	 * TODO: Fill-in other fields of iolink subtype
1103 	 */
1104 	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
1105 	sub_type_hdr->proximity_domain_from = proximity_domain;
1106 #ifdef CONFIG_NUMA
1107 	if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
1108 		sub_type_hdr->proximity_domain_to = 0;
1109 	else
1110 		sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node;
1111 #else
1112 	sub_type_hdr->proximity_domain_to = 0;
1113 #endif
1114 	return 0;
1115 }
1116 
kfd_fill_gpu_xgmi_link_to_gpu(int * avail_size,struct kfd_dev * kdev,struct kfd_dev * peer_kdev,struct crat_subtype_iolink * sub_type_hdr,uint32_t proximity_domain_from,uint32_t proximity_domain_to)1117 static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
1118 			struct kfd_dev *kdev,
1119 			struct kfd_dev *peer_kdev,
1120 			struct crat_subtype_iolink *sub_type_hdr,
1121 			uint32_t proximity_domain_from,
1122 			uint32_t proximity_domain_to)
1123 {
1124 	*avail_size -= sizeof(struct crat_subtype_iolink);
1125 	if (*avail_size < 0)
1126 		return -ENOMEM;
1127 
1128 	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1129 
1130 	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1131 	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1132 	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED |
1133 			       CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1134 
1135 	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
1136 	sub_type_hdr->proximity_domain_from = proximity_domain_from;
1137 	sub_type_hdr->proximity_domain_to = proximity_domain_to;
1138 	sub_type_hdr->num_hops_xgmi =
1139 		amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
1140 	return 0;
1141 }
1142 
1143 /* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
1144  *
1145  *	@pcrat_image: Fill in VCRAT for GPU
1146  *	@size:	[IN] allocated size of crat_image.
1147  *		[OUT] actual size of data filled in crat_image
1148  */
kfd_create_vcrat_image_gpu(void * pcrat_image,size_t * size,struct kfd_dev * kdev,uint32_t proximity_domain)1149 static int kfd_create_vcrat_image_gpu(void *pcrat_image,
1150 				      size_t *size, struct kfd_dev *kdev,
1151 				      uint32_t proximity_domain)
1152 {
1153 	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
1154 	struct crat_subtype_generic *sub_type_hdr;
1155 	struct kfd_local_mem_info local_mem_info;
1156 	struct kfd_topology_device *peer_dev;
1157 	struct crat_subtype_computeunit *cu;
1158 	struct kfd_cu_info cu_info;
1159 	int avail_size = *size;
1160 	uint32_t total_num_of_cu;
1161 	int num_of_cache_entries = 0;
1162 	int cache_mem_filled = 0;
1163 	uint32_t nid = 0;
1164 	int ret = 0;
1165 
1166 	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
1167 		return -EINVAL;
1168 
1169 	/* Fill the CRAT Header.
1170 	 * Modify length and total_entries as subunits are added.
1171 	 */
1172 	avail_size -= sizeof(struct crat_header);
1173 	if (avail_size < 0)
1174 		return -ENOMEM;
1175 
1176 	memset(crat_table, 0, sizeof(struct crat_header));
1177 
1178 	memcpy(&crat_table->signature, CRAT_SIGNATURE,
1179 			sizeof(crat_table->signature));
1180 	/* Change length as we add more subtypes*/
1181 	crat_table->length = sizeof(struct crat_header);
1182 	crat_table->num_domains = 1;
1183 	crat_table->total_entries = 0;
1184 
1185 	/* Fill in Subtype: Compute Unit
1186 	 * First fill in the sub type header and then sub type data
1187 	 */
1188 	avail_size -= sizeof(struct crat_subtype_computeunit);
1189 	if (avail_size < 0)
1190 		return -ENOMEM;
1191 
1192 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
1193 	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
1194 
1195 	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
1196 	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
1197 	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
1198 
1199 	/* Fill CU subtype data */
1200 	cu = (struct crat_subtype_computeunit *)sub_type_hdr;
1201 	cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
1202 	cu->proximity_domain = proximity_domain;
1203 
1204 	amdgpu_amdkfd_get_cu_info(kdev->kgd, &cu_info);
1205 	cu->num_simd_per_cu = cu_info.simd_per_cu;
1206 	cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
1207 	cu->max_waves_simd = cu_info.max_waves_per_simd;
1208 
1209 	cu->wave_front_size = cu_info.wave_front_size;
1210 	cu->array_count = cu_info.num_shader_arrays_per_engine *
1211 		cu_info.num_shader_engines;
1212 	total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
1213 	cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
1214 	cu->num_cu_per_array = cu_info.num_cu_per_sh;
1215 	cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
1216 	cu->num_banks = cu_info.num_shader_engines;
1217 	cu->lds_size_in_kb = cu_info.lds_size;
1218 
1219 	cu->hsa_capability = 0;
1220 
1221 	/* Check if this node supports IOMMU. During parsing this flag will
1222 	 * translate to HSA_CAP_ATS_PRESENT
1223 	 */
1224 	if (!kfd_iommu_check_device(kdev))
1225 		cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
1226 
1227 	crat_table->length += sub_type_hdr->length;
1228 	crat_table->total_entries++;
1229 
1230 	/* Fill in Subtype: Memory. Only on systems with large BAR (no
1231 	 * private FB), report memory as public. On other systems
1232 	 * report the total FB size (public+private) as a single
1233 	 * private heap.
1234 	 */
1235 	amdgpu_amdkfd_get_local_mem_info(kdev->kgd, &local_mem_info);
1236 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1237 			sub_type_hdr->length);
1238 
1239 	if (debug_largebar)
1240 		local_mem_info.local_mem_size_private = 0;
1241 
1242 	if (local_mem_info.local_mem_size_private == 0)
1243 		ret = kfd_fill_gpu_memory_affinity(&avail_size,
1244 				kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
1245 				local_mem_info.local_mem_size_public,
1246 				(struct crat_subtype_memory *)sub_type_hdr,
1247 				proximity_domain,
1248 				&local_mem_info);
1249 	else
1250 		ret = kfd_fill_gpu_memory_affinity(&avail_size,
1251 				kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
1252 				local_mem_info.local_mem_size_public +
1253 				local_mem_info.local_mem_size_private,
1254 				(struct crat_subtype_memory *)sub_type_hdr,
1255 				proximity_domain,
1256 				&local_mem_info);
1257 	if (ret < 0)
1258 		return ret;
1259 
1260 	crat_table->length += sizeof(struct crat_subtype_memory);
1261 	crat_table->total_entries++;
1262 
1263 	/* TODO: Fill in cache information. This information is NOT readily
1264 	 * available in KGD
1265 	 */
1266 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1267 		sub_type_hdr->length);
1268 	ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
1269 				avail_size,
1270 				&cu_info,
1271 				(struct crat_subtype_cache *)sub_type_hdr,
1272 				&cache_mem_filled,
1273 				&num_of_cache_entries);
1274 
1275 	if (ret < 0)
1276 		return ret;
1277 
1278 	crat_table->length += cache_mem_filled;
1279 	crat_table->total_entries += num_of_cache_entries;
1280 	avail_size -= cache_mem_filled;
1281 
1282 	/* Fill in Subtype: IO_LINKS
1283 	 *  Only direct links are added here which is Link from GPU to
1284 	 *  to its NUMA node. Indirect links are added by userspace.
1285 	 */
1286 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1287 		cache_mem_filled);
1288 	ret = kfd_fill_gpu_direct_io_link_to_cpu(&avail_size, kdev,
1289 		(struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
1290 
1291 	if (ret < 0)
1292 		return ret;
1293 
1294 	crat_table->length += sub_type_hdr->length;
1295 	crat_table->total_entries++;
1296 
1297 
1298 	/* Fill in Subtype: IO_LINKS
1299 	 * Direct links from GPU to other GPUs through xGMI.
1300 	 * We will loop GPUs that already be processed (with lower value
1301 	 * of proximity_domain), add the link for the GPUs with same
1302 	 * hive id (from this GPU to other GPU) . The reversed iolink
1303 	 * (from other GPU to this GPU) will be added
1304 	 * in kfd_parse_subtype_iolink.
1305 	 */
1306 	if (kdev->hive_id) {
1307 		for (nid = 0; nid < proximity_domain; ++nid) {
1308 			peer_dev = kfd_topology_device_by_proximity_domain(nid);
1309 			if (!peer_dev->gpu)
1310 				continue;
1311 			if (peer_dev->gpu->hive_id != kdev->hive_id)
1312 				continue;
1313 			sub_type_hdr = (typeof(sub_type_hdr))(
1314 				(char *)sub_type_hdr +
1315 				sizeof(struct crat_subtype_iolink));
1316 			ret = kfd_fill_gpu_xgmi_link_to_gpu(
1317 				&avail_size, kdev, peer_dev->gpu,
1318 				(struct crat_subtype_iolink *)sub_type_hdr,
1319 				proximity_domain, nid);
1320 			if (ret < 0)
1321 				return ret;
1322 			crat_table->length += sub_type_hdr->length;
1323 			crat_table->total_entries++;
1324 		}
1325 	}
1326 	*size = crat_table->length;
1327 	pr_info("Virtual CRAT table created for GPU\n");
1328 
1329 	return ret;
1330 }
1331 
1332 /* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
1333  *		creates a Virtual CRAT (VCRAT) image
1334  *
1335  * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
1336  *
1337  *	@crat_image: VCRAT image created because ACPI does not have a
1338  *		     CRAT for this device
1339  *	@size: [OUT] size of virtual crat_image
1340  *	@flags:	COMPUTE_UNIT_CPU - Create VCRAT for CPU device
1341  *		COMPUTE_UNIT_GPU - Create VCRAT for GPU
1342  *		(COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
1343  *			-- this option is not currently implemented.
1344  *			The assumption is that all AMD APUs will have CRAT
1345  *	@kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
1346  *
1347  *	Return 0 if successful else return -ve value
1348  */
kfd_create_crat_image_virtual(void ** crat_image,size_t * size,int flags,struct kfd_dev * kdev,uint32_t proximity_domain)1349 int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
1350 				  int flags, struct kfd_dev *kdev,
1351 				  uint32_t proximity_domain)
1352 {
1353 	void *pcrat_image = NULL;
1354 	int ret = 0;
1355 
1356 	if (!crat_image)
1357 		return -EINVAL;
1358 
1359 	*crat_image = NULL;
1360 
1361 	/* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and
1362 	 * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover
1363 	 * all the current conditions. A check is put not to overwrite beyond
1364 	 * allocated size
1365 	 */
1366 	switch (flags) {
1367 	case COMPUTE_UNIT_CPU:
1368 		pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL);
1369 		if (!pcrat_image)
1370 			return -ENOMEM;
1371 		*size = VCRAT_SIZE_FOR_CPU;
1372 		ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
1373 		break;
1374 	case COMPUTE_UNIT_GPU:
1375 		if (!kdev)
1376 			return -EINVAL;
1377 		pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
1378 		if (!pcrat_image)
1379 			return -ENOMEM;
1380 		*size = VCRAT_SIZE_FOR_GPU;
1381 		ret = kfd_create_vcrat_image_gpu(pcrat_image, size, kdev,
1382 						 proximity_domain);
1383 		break;
1384 	case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU):
1385 		/* TODO: */
1386 		ret = -EINVAL;
1387 		pr_err("VCRAT not implemented for APU\n");
1388 		break;
1389 	default:
1390 		ret = -EINVAL;
1391 	}
1392 
1393 	if (!ret)
1394 		*crat_image = pcrat_image;
1395 	else
1396 		kfree(pcrat_image);
1397 
1398 	return ret;
1399 }
1400 
1401 
1402 /* kfd_destroy_crat_image
1403  *
1404  *	@crat_image: [IN] - crat_image from kfd_create_crat_image_xxx(..)
1405  *
1406  */
kfd_destroy_crat_image(void * crat_image)1407 void kfd_destroy_crat_image(void *crat_image)
1408 {
1409 	kfree(crat_image);
1410 }
1411