1 /* $NetBSD: amdgpu_debugfs.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $ */
2
3 /*
4 * Copyright 2008 Advanced Micro Devices, Inc.
5 * Copyright 2008 Red Hat Inc.
6 * Copyright 2009 Jerome Glisse.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
25 *
26 */
27
28 #include <sys/cdefs.h>
29 __KERNEL_RCSID(0, "$NetBSD: amdgpu_debugfs.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $");
30
31 #include <linux/kthread.h>
32 #include <linux/pci.h>
33 #include <linux/uaccess.h>
34 #include <linux/pm_runtime.h>
35
36 #include <drm/drm_debugfs.h>
37
38 #include "amdgpu.h"
39
40 /**
41 * amdgpu_debugfs_add_files - Add simple debugfs entries
42 *
43 * @adev: Device to attach debugfs entries to
44 * @files: Array of function callbacks that respond to reads
45 * @nfiles: Number of callbacks to register
46 *
47 */
amdgpu_debugfs_add_files(struct amdgpu_device * adev,const struct drm_info_list * files,unsigned nfiles)48 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
49 const struct drm_info_list *files,
50 unsigned nfiles)
51 {
52 unsigned i;
53
54 for (i = 0; i < adev->debugfs_count; i++) {
55 if (adev->debugfs[i].files == files) {
56 /* Already registered */
57 return 0;
58 }
59 }
60
61 i = adev->debugfs_count + 1;
62 if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
63 DRM_ERROR("Reached maximum number of debugfs components.\n");
64 DRM_ERROR("Report so we increase "
65 "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
66 return -EINVAL;
67 }
68 adev->debugfs[adev->debugfs_count].files = files;
69 adev->debugfs[adev->debugfs_count].num_files = nfiles;
70 adev->debugfs_count = i;
71 #if defined(CONFIG_DEBUG_FS)
72 drm_debugfs_create_files(files, nfiles,
73 adev->ddev->primary->debugfs_root,
74 adev->ddev->primary);
75 #endif
76 return 0;
77 }
78
79 #if defined(CONFIG_DEBUG_FS)
80
81 /**
82 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
83 *
84 * @read: True if reading
85 * @f: open file handle
86 * @buf: User buffer to write/read to
87 * @size: Number of bytes to write/read
88 * @pos: Offset to seek to
89 *
90 * This debugfs entry has special meaning on the offset being sought.
91 * Various bits have different meanings:
92 *
93 * Bit 62: Indicates a GRBM bank switch is needed
94 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
95 * zero)
96 * Bits 24..33: The SE or ME selector if needed
97 * Bits 34..43: The SH (or SA) or PIPE selector if needed
98 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
99 *
100 * Bit 23: Indicates that the PM power gating lock should be held
101 * This is necessary to read registers that might be
102 * unreliable during a power gating transistion.
103 *
104 * The lower bits are the BYTE offset of the register to read. This
105 * allows reading multiple registers in a single call and having
106 * the returned size reflect that.
107 */
amdgpu_debugfs_process_reg_op(bool read,struct file * f,char __user * buf,size_t size,loff_t * pos)108 static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
109 char __user *buf, size_t size, loff_t *pos)
110 {
111 struct amdgpu_device *adev = file_inode(f)->i_private;
112 ssize_t result = 0;
113 int r;
114 bool pm_pg_lock, use_bank, use_ring;
115 unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
116
117 pm_pg_lock = use_bank = use_ring = false;
118 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
119
120 if (size & 0x3 || *pos & 0x3 ||
121 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
122 return -EINVAL;
123
124 /* are we reading registers for which a PG lock is necessary? */
125 pm_pg_lock = (*pos >> 23) & 1;
126
127 if (*pos & (1ULL << 62)) {
128 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
129 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
130 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
131
132 if (se_bank == 0x3FF)
133 se_bank = 0xFFFFFFFF;
134 if (sh_bank == 0x3FF)
135 sh_bank = 0xFFFFFFFF;
136 if (instance_bank == 0x3FF)
137 instance_bank = 0xFFFFFFFF;
138 use_bank = true;
139 } else if (*pos & (1ULL << 61)) {
140
141 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
142 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
143 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
144 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
145
146 use_ring = true;
147 } else {
148 use_bank = use_ring = false;
149 }
150
151 *pos &= (1UL << 22) - 1;
152
153 r = pm_runtime_get_sync(adev->ddev->dev);
154 if (r < 0)
155 return r;
156
157 if (use_bank) {
158 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
159 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
160 pm_runtime_mark_last_busy(adev->ddev->dev);
161 pm_runtime_put_autosuspend(adev->ddev->dev);
162 return -EINVAL;
163 }
164 mutex_lock(&adev->grbm_idx_mutex);
165 amdgpu_gfx_select_se_sh(adev, se_bank,
166 sh_bank, instance_bank);
167 } else if (use_ring) {
168 mutex_lock(&adev->srbm_mutex);
169 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
170 }
171
172 if (pm_pg_lock)
173 mutex_lock(&adev->pm.mutex);
174
175 while (size) {
176 uint32_t value;
177
178 if (read) {
179 value = RREG32(*pos >> 2);
180 r = put_user(value, (uint32_t *)buf);
181 } else {
182 r = get_user(value, (uint32_t *)buf);
183 if (!r)
184 WREG32(*pos >> 2, value);
185 }
186 if (r) {
187 result = r;
188 goto end;
189 }
190
191 result += 4;
192 buf += 4;
193 *pos += 4;
194 size -= 4;
195 }
196
197 end:
198 if (use_bank) {
199 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
200 mutex_unlock(&adev->grbm_idx_mutex);
201 } else if (use_ring) {
202 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
203 mutex_unlock(&adev->srbm_mutex);
204 }
205
206 if (pm_pg_lock)
207 mutex_unlock(&adev->pm.mutex);
208
209 pm_runtime_mark_last_busy(adev->ddev->dev);
210 pm_runtime_put_autosuspend(adev->ddev->dev);
211
212 return result;
213 }
214
215 /**
216 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
217 */
amdgpu_debugfs_regs_read(struct file * f,char __user * buf,size_t size,loff_t * pos)218 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
219 size_t size, loff_t *pos)
220 {
221 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
222 }
223
224 /**
225 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
226 */
amdgpu_debugfs_regs_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)227 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
228 size_t size, loff_t *pos)
229 {
230 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
231 }
232
233
234 /**
235 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
236 *
237 * @f: open file handle
238 * @buf: User buffer to store read data in
239 * @size: Number of bytes to read
240 * @pos: Offset to seek to
241 *
242 * The lower bits are the BYTE offset of the register to read. This
243 * allows reading multiple registers in a single call and having
244 * the returned size reflect that.
245 */
amdgpu_debugfs_regs_pcie_read(struct file * f,char __user * buf,size_t size,loff_t * pos)246 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
247 size_t size, loff_t *pos)
248 {
249 struct amdgpu_device *adev = file_inode(f)->i_private;
250 ssize_t result = 0;
251 int r;
252
253 if (size & 0x3 || *pos & 0x3)
254 return -EINVAL;
255
256 r = pm_runtime_get_sync(adev->ddev->dev);
257 if (r < 0)
258 return r;
259
260 while (size) {
261 uint32_t value;
262
263 value = RREG32_PCIE(*pos >> 2);
264 r = put_user(value, (uint32_t *)buf);
265 if (r) {
266 pm_runtime_mark_last_busy(adev->ddev->dev);
267 pm_runtime_put_autosuspend(adev->ddev->dev);
268 return r;
269 }
270
271 result += 4;
272 buf += 4;
273 *pos += 4;
274 size -= 4;
275 }
276
277 pm_runtime_mark_last_busy(adev->ddev->dev);
278 pm_runtime_put_autosuspend(adev->ddev->dev);
279
280 return result;
281 }
282
283 /**
284 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
285 *
286 * @f: open file handle
287 * @buf: User buffer to write data from
288 * @size: Number of bytes to write
289 * @pos: Offset to seek to
290 *
291 * The lower bits are the BYTE offset of the register to write. This
292 * allows writing multiple registers in a single call and having
293 * the returned size reflect that.
294 */
amdgpu_debugfs_regs_pcie_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)295 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
296 size_t size, loff_t *pos)
297 {
298 struct amdgpu_device *adev = file_inode(f)->i_private;
299 ssize_t result = 0;
300 int r;
301
302 if (size & 0x3 || *pos & 0x3)
303 return -EINVAL;
304
305 r = pm_runtime_get_sync(adev->ddev->dev);
306 if (r < 0)
307 return r;
308
309 while (size) {
310 uint32_t value;
311
312 r = get_user(value, (uint32_t *)buf);
313 if (r) {
314 pm_runtime_mark_last_busy(adev->ddev->dev);
315 pm_runtime_put_autosuspend(adev->ddev->dev);
316 return r;
317 }
318
319 WREG32_PCIE(*pos >> 2, value);
320
321 result += 4;
322 buf += 4;
323 *pos += 4;
324 size -= 4;
325 }
326
327 pm_runtime_mark_last_busy(adev->ddev->dev);
328 pm_runtime_put_autosuspend(adev->ddev->dev);
329
330 return result;
331 }
332
333 /**
334 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
335 *
336 * @f: open file handle
337 * @buf: User buffer to store read data in
338 * @size: Number of bytes to read
339 * @pos: Offset to seek to
340 *
341 * The lower bits are the BYTE offset of the register to read. This
342 * allows reading multiple registers in a single call and having
343 * the returned size reflect that.
344 */
amdgpu_debugfs_regs_didt_read(struct file * f,char __user * buf,size_t size,loff_t * pos)345 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
346 size_t size, loff_t *pos)
347 {
348 struct amdgpu_device *adev = file_inode(f)->i_private;
349 ssize_t result = 0;
350 int r;
351
352 if (size & 0x3 || *pos & 0x3)
353 return -EINVAL;
354
355 r = pm_runtime_get_sync(adev->ddev->dev);
356 if (r < 0)
357 return r;
358
359 while (size) {
360 uint32_t value;
361
362 value = RREG32_DIDT(*pos >> 2);
363 r = put_user(value, (uint32_t *)buf);
364 if (r) {
365 pm_runtime_mark_last_busy(adev->ddev->dev);
366 pm_runtime_put_autosuspend(adev->ddev->dev);
367 return r;
368 }
369
370 result += 4;
371 buf += 4;
372 *pos += 4;
373 size -= 4;
374 }
375
376 pm_runtime_mark_last_busy(adev->ddev->dev);
377 pm_runtime_put_autosuspend(adev->ddev->dev);
378
379 return result;
380 }
381
382 /**
383 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
384 *
385 * @f: open file handle
386 * @buf: User buffer to write data from
387 * @size: Number of bytes to write
388 * @pos: Offset to seek to
389 *
390 * The lower bits are the BYTE offset of the register to write. This
391 * allows writing multiple registers in a single call and having
392 * the returned size reflect that.
393 */
amdgpu_debugfs_regs_didt_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)394 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
395 size_t size, loff_t *pos)
396 {
397 struct amdgpu_device *adev = file_inode(f)->i_private;
398 ssize_t result = 0;
399 int r;
400
401 if (size & 0x3 || *pos & 0x3)
402 return -EINVAL;
403
404 r = pm_runtime_get_sync(adev->ddev->dev);
405 if (r < 0)
406 return r;
407
408 while (size) {
409 uint32_t value;
410
411 r = get_user(value, (uint32_t *)buf);
412 if (r) {
413 pm_runtime_mark_last_busy(adev->ddev->dev);
414 pm_runtime_put_autosuspend(adev->ddev->dev);
415 return r;
416 }
417
418 WREG32_DIDT(*pos >> 2, value);
419
420 result += 4;
421 buf += 4;
422 *pos += 4;
423 size -= 4;
424 }
425
426 pm_runtime_mark_last_busy(adev->ddev->dev);
427 pm_runtime_put_autosuspend(adev->ddev->dev);
428
429 return result;
430 }
431
432 /**
433 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
434 *
435 * @f: open file handle
436 * @buf: User buffer to store read data in
437 * @size: Number of bytes to read
438 * @pos: Offset to seek to
439 *
440 * The lower bits are the BYTE offset of the register to read. This
441 * allows reading multiple registers in a single call and having
442 * the returned size reflect that.
443 */
amdgpu_debugfs_regs_smc_read(struct file * f,char __user * buf,size_t size,loff_t * pos)444 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
445 size_t size, loff_t *pos)
446 {
447 struct amdgpu_device *adev = file_inode(f)->i_private;
448 ssize_t result = 0;
449 int r;
450
451 if (size & 0x3 || *pos & 0x3)
452 return -EINVAL;
453
454 r = pm_runtime_get_sync(adev->ddev->dev);
455 if (r < 0)
456 return r;
457
458 while (size) {
459 uint32_t value;
460
461 value = RREG32_SMC(*pos);
462 r = put_user(value, (uint32_t *)buf);
463 if (r) {
464 pm_runtime_mark_last_busy(adev->ddev->dev);
465 pm_runtime_put_autosuspend(adev->ddev->dev);
466 return r;
467 }
468
469 result += 4;
470 buf += 4;
471 *pos += 4;
472 size -= 4;
473 }
474
475 pm_runtime_mark_last_busy(adev->ddev->dev);
476 pm_runtime_put_autosuspend(adev->ddev->dev);
477
478 return result;
479 }
480
481 /**
482 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
483 *
484 * @f: open file handle
485 * @buf: User buffer to write data from
486 * @size: Number of bytes to write
487 * @pos: Offset to seek to
488 *
489 * The lower bits are the BYTE offset of the register to write. This
490 * allows writing multiple registers in a single call and having
491 * the returned size reflect that.
492 */
amdgpu_debugfs_regs_smc_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)493 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
494 size_t size, loff_t *pos)
495 {
496 struct amdgpu_device *adev = file_inode(f)->i_private;
497 ssize_t result = 0;
498 int r;
499
500 if (size & 0x3 || *pos & 0x3)
501 return -EINVAL;
502
503 r = pm_runtime_get_sync(adev->ddev->dev);
504 if (r < 0)
505 return r;
506
507 while (size) {
508 uint32_t value;
509
510 r = get_user(value, (uint32_t *)buf);
511 if (r) {
512 pm_runtime_mark_last_busy(adev->ddev->dev);
513 pm_runtime_put_autosuspend(adev->ddev->dev);
514 return r;
515 }
516
517 WREG32_SMC(*pos, value);
518
519 result += 4;
520 buf += 4;
521 *pos += 4;
522 size -= 4;
523 }
524
525 pm_runtime_mark_last_busy(adev->ddev->dev);
526 pm_runtime_put_autosuspend(adev->ddev->dev);
527
528 return result;
529 }
530
531 /**
532 * amdgpu_debugfs_gca_config_read - Read from gfx config data
533 *
534 * @f: open file handle
535 * @buf: User buffer to store read data in
536 * @size: Number of bytes to read
537 * @pos: Offset to seek to
538 *
539 * This file is used to access configuration data in a somewhat
540 * stable fashion. The format is a series of DWORDs with the first
541 * indicating which revision it is. New content is appended to the
542 * end so that older software can still read the data.
543 */
544
amdgpu_debugfs_gca_config_read(struct file * f,char __user * buf,size_t size,loff_t * pos)545 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
546 size_t size, loff_t *pos)
547 {
548 struct amdgpu_device *adev = file_inode(f)->i_private;
549 ssize_t result = 0;
550 int r;
551 uint32_t *config, no_regs = 0;
552
553 if (size & 0x3 || *pos & 0x3)
554 return -EINVAL;
555
556 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
557 if (!config)
558 return -ENOMEM;
559
560 /* version, increment each time something is added */
561 config[no_regs++] = 3;
562 config[no_regs++] = adev->gfx.config.max_shader_engines;
563 config[no_regs++] = adev->gfx.config.max_tile_pipes;
564 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
565 config[no_regs++] = adev->gfx.config.max_sh_per_se;
566 config[no_regs++] = adev->gfx.config.max_backends_per_se;
567 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
568 config[no_regs++] = adev->gfx.config.max_gprs;
569 config[no_regs++] = adev->gfx.config.max_gs_threads;
570 config[no_regs++] = adev->gfx.config.max_hw_contexts;
571 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
572 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
573 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
574 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
575 config[no_regs++] = adev->gfx.config.num_tile_pipes;
576 config[no_regs++] = adev->gfx.config.backend_enable_mask;
577 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
578 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
579 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
580 config[no_regs++] = adev->gfx.config.num_gpus;
581 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
582 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
583 config[no_regs++] = adev->gfx.config.gb_addr_config;
584 config[no_regs++] = adev->gfx.config.num_rbs;
585
586 /* rev==1 */
587 config[no_regs++] = adev->rev_id;
588 config[no_regs++] = adev->pg_flags;
589 config[no_regs++] = adev->cg_flags;
590
591 /* rev==2 */
592 config[no_regs++] = adev->family;
593 config[no_regs++] = adev->external_rev_id;
594
595 /* rev==3 */
596 config[no_regs++] = adev->pdev->device;
597 config[no_regs++] = adev->pdev->revision;
598 config[no_regs++] = adev->pdev->subsystem_device;
599 config[no_regs++] = adev->pdev->subsystem_vendor;
600
601 while (size && (*pos < no_regs * 4)) {
602 uint32_t value;
603
604 value = config[*pos >> 2];
605 r = put_user(value, (uint32_t *)buf);
606 if (r) {
607 kfree(config);
608 return r;
609 }
610
611 result += 4;
612 buf += 4;
613 *pos += 4;
614 size -= 4;
615 }
616
617 kfree(config);
618 return result;
619 }
620
621 /**
622 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
623 *
624 * @f: open file handle
625 * @buf: User buffer to store read data in
626 * @size: Number of bytes to read
627 * @pos: Offset to seek to
628 *
629 * The offset is treated as the BYTE address of one of the sensors
630 * enumerated in amd/include/kgd_pp_interface.h under the
631 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
632 * you would use the offset 3 * 4 = 12.
633 */
amdgpu_debugfs_sensor_read(struct file * f,char __user * buf,size_t size,loff_t * pos)634 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
635 size_t size, loff_t *pos)
636 {
637 struct amdgpu_device *adev = file_inode(f)->i_private;
638 int idx, x, outsize, r, valuesize;
639 uint32_t values[16];
640
641 if (size & 3 || *pos & 0x3)
642 return -EINVAL;
643
644 if (!adev->pm.dpm_enabled)
645 return -EINVAL;
646
647 /* convert offset to sensor number */
648 idx = *pos >> 2;
649
650 valuesize = sizeof(values);
651
652 r = pm_runtime_get_sync(adev->ddev->dev);
653 if (r < 0)
654 return r;
655
656 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
657
658 pm_runtime_mark_last_busy(adev->ddev->dev);
659 pm_runtime_put_autosuspend(adev->ddev->dev);
660
661 if (r)
662 return r;
663
664 if (size > valuesize)
665 return -EINVAL;
666
667 outsize = 0;
668 x = 0;
669 if (!r) {
670 while (size) {
671 r = put_user(values[x++], (int32_t *)buf);
672 buf += 4;
673 size -= 4;
674 outsize += 4;
675 }
676 }
677
678 return !r ? outsize : r;
679 }
680
681 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
682 *
683 * @f: open file handle
684 * @buf: User buffer to store read data in
685 * @size: Number of bytes to read
686 * @pos: Offset to seek to
687 *
688 * The offset being sought changes which wave that the status data
689 * will be returned for. The bits are used as follows:
690 *
691 * Bits 0..6: Byte offset into data
692 * Bits 7..14: SE selector
693 * Bits 15..22: SH/SA selector
694 * Bits 23..30: CU/{WGP+SIMD} selector
695 * Bits 31..36: WAVE ID selector
696 * Bits 37..44: SIMD ID selector
697 *
698 * The returned data begins with one DWORD of version information
699 * Followed by WAVE STATUS registers relevant to the GFX IP version
700 * being used. See gfx_v8_0_read_wave_data() for an example output.
701 */
amdgpu_debugfs_wave_read(struct file * f,char __user * buf,size_t size,loff_t * pos)702 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
703 size_t size, loff_t *pos)
704 {
705 struct amdgpu_device *adev = f->f_inode->i_private;
706 int r, x;
707 ssize_t result=0;
708 uint32_t offset, se, sh, cu, wave, simd, data[32];
709
710 if (size & 3 || *pos & 3)
711 return -EINVAL;
712
713 /* decode offset */
714 offset = (*pos & GENMASK_ULL(6, 0));
715 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
716 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
717 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
718 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
719 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
720
721 r = pm_runtime_get_sync(adev->ddev->dev);
722 if (r < 0)
723 return r;
724
725 /* switch to the specific se/sh/cu */
726 mutex_lock(&adev->grbm_idx_mutex);
727 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
728
729 x = 0;
730 if (adev->gfx.funcs->read_wave_data)
731 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
732
733 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
734 mutex_unlock(&adev->grbm_idx_mutex);
735
736 pm_runtime_mark_last_busy(adev->ddev->dev);
737 pm_runtime_put_autosuspend(adev->ddev->dev);
738
739 if (!x)
740 return -EINVAL;
741
742 while (size && (offset < x * 4)) {
743 uint32_t value;
744
745 value = data[offset >> 2];
746 r = put_user(value, (uint32_t *)buf);
747 if (r)
748 return r;
749
750 result += 4;
751 buf += 4;
752 offset += 4;
753 size -= 4;
754 }
755
756 return result;
757 }
758
759 /** amdgpu_debugfs_gpr_read - Read wave gprs
760 *
761 * @f: open file handle
762 * @buf: User buffer to store read data in
763 * @size: Number of bytes to read
764 * @pos: Offset to seek to
765 *
766 * The offset being sought changes which wave that the status data
767 * will be returned for. The bits are used as follows:
768 *
769 * Bits 0..11: Byte offset into data
770 * Bits 12..19: SE selector
771 * Bits 20..27: SH/SA selector
772 * Bits 28..35: CU/{WGP+SIMD} selector
773 * Bits 36..43: WAVE ID selector
774 * Bits 37..44: SIMD ID selector
775 * Bits 52..59: Thread selector
776 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
777 *
778 * The return data comes from the SGPR or VGPR register bank for
779 * the selected operational unit.
780 */
amdgpu_debugfs_gpr_read(struct file * f,char __user * buf,size_t size,loff_t * pos)781 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
782 size_t size, loff_t *pos)
783 {
784 struct amdgpu_device *adev = f->f_inode->i_private;
785 int r;
786 ssize_t result = 0;
787 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
788
789 if (size & 3 || *pos & 3)
790 return -EINVAL;
791
792 /* decode offset */
793 offset = *pos & GENMASK_ULL(11, 0);
794 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
795 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
796 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
797 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
798 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
799 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
800 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
801
802 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
803 if (!data)
804 return -ENOMEM;
805
806 r = pm_runtime_get_sync(adev->ddev->dev);
807 if (r < 0)
808 return r;
809
810 /* switch to the specific se/sh/cu */
811 mutex_lock(&adev->grbm_idx_mutex);
812 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
813
814 if (bank == 0) {
815 if (adev->gfx.funcs->read_wave_vgprs)
816 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
817 } else {
818 if (adev->gfx.funcs->read_wave_sgprs)
819 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
820 }
821
822 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
823 mutex_unlock(&adev->grbm_idx_mutex);
824
825 pm_runtime_mark_last_busy(adev->ddev->dev);
826 pm_runtime_put_autosuspend(adev->ddev->dev);
827
828 while (size) {
829 uint32_t value;
830
831 value = data[offset++];
832 r = put_user(value, (uint32_t *)buf);
833 if (r) {
834 result = r;
835 goto err;
836 }
837
838 result += 4;
839 buf += 4;
840 size -= 4;
841 }
842
843 err:
844 kfree(data);
845 return result;
846 }
847
848 static const struct file_operations amdgpu_debugfs_regs_fops = {
849 .owner = THIS_MODULE,
850 .read = amdgpu_debugfs_regs_read,
851 .write = amdgpu_debugfs_regs_write,
852 .llseek = default_llseek
853 };
854 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
855 .owner = THIS_MODULE,
856 .read = amdgpu_debugfs_regs_didt_read,
857 .write = amdgpu_debugfs_regs_didt_write,
858 .llseek = default_llseek
859 };
860 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
861 .owner = THIS_MODULE,
862 .read = amdgpu_debugfs_regs_pcie_read,
863 .write = amdgpu_debugfs_regs_pcie_write,
864 .llseek = default_llseek
865 };
866 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
867 .owner = THIS_MODULE,
868 .read = amdgpu_debugfs_regs_smc_read,
869 .write = amdgpu_debugfs_regs_smc_write,
870 .llseek = default_llseek
871 };
872
873 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
874 .owner = THIS_MODULE,
875 .read = amdgpu_debugfs_gca_config_read,
876 .llseek = default_llseek
877 };
878
879 static const struct file_operations amdgpu_debugfs_sensors_fops = {
880 .owner = THIS_MODULE,
881 .read = amdgpu_debugfs_sensor_read,
882 .llseek = default_llseek
883 };
884
885 static const struct file_operations amdgpu_debugfs_wave_fops = {
886 .owner = THIS_MODULE,
887 .read = amdgpu_debugfs_wave_read,
888 .llseek = default_llseek
889 };
890 static const struct file_operations amdgpu_debugfs_gpr_fops = {
891 .owner = THIS_MODULE,
892 .read = amdgpu_debugfs_gpr_read,
893 .llseek = default_llseek
894 };
895
896 static const struct file_operations *debugfs_regs[] = {
897 &amdgpu_debugfs_regs_fops,
898 &amdgpu_debugfs_regs_didt_fops,
899 &amdgpu_debugfs_regs_pcie_fops,
900 &amdgpu_debugfs_regs_smc_fops,
901 &amdgpu_debugfs_gca_config_fops,
902 &amdgpu_debugfs_sensors_fops,
903 &amdgpu_debugfs_wave_fops,
904 &amdgpu_debugfs_gpr_fops,
905 };
906
907 static const char *debugfs_regs_names[] = {
908 "amdgpu_regs",
909 "amdgpu_regs_didt",
910 "amdgpu_regs_pcie",
911 "amdgpu_regs_smc",
912 "amdgpu_gca_config",
913 "amdgpu_sensors",
914 "amdgpu_wave",
915 "amdgpu_gpr",
916 };
917
918 /**
919 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
920 * register access.
921 *
922 * @adev: The device to attach the debugfs entries to
923 */
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)924 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
925 {
926 struct drm_minor *minor = adev->ddev->primary;
927 struct dentry *ent, *root = minor->debugfs_root;
928 unsigned int i;
929
930 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
931 ent = debugfs_create_file(debugfs_regs_names[i],
932 S_IFREG | S_IRUGO, root,
933 adev, debugfs_regs[i]);
934 if (!i && !IS_ERR_OR_NULL(ent))
935 i_size_write(ent->d_inode, adev->rmmio_size);
936 adev->debugfs_regs[i] = ent;
937 }
938
939 return 0;
940 }
941
amdgpu_debugfs_regs_cleanup(struct amdgpu_device * adev)942 void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
943 {
944 unsigned i;
945
946 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
947 if (adev->debugfs_regs[i]) {
948 debugfs_remove(adev->debugfs_regs[i]);
949 adev->debugfs_regs[i] = NULL;
950 }
951 }
952 }
953
amdgpu_debugfs_test_ib(struct seq_file * m,void * data)954 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
955 {
956 struct drm_info_node *node = (struct drm_info_node *) m->private;
957 struct drm_device *dev = node->minor->dev;
958 struct amdgpu_device *adev = dev->dev_private;
959 int r = 0, i;
960
961 r = pm_runtime_get_sync(dev->dev);
962 if (r < 0)
963 return r;
964
965 /* Avoid accidently unparking the sched thread during GPU reset */
966 mutex_lock(&adev->lock_reset);
967
968 /* hold on the scheduler */
969 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
970 struct amdgpu_ring *ring = adev->rings[i];
971
972 if (!ring || !ring->sched.thread)
973 continue;
974 kthread_park(ring->sched.thread);
975 }
976
977 seq_printf(m, "run ib test:\n");
978 r = amdgpu_ib_ring_tests(adev);
979 if (r)
980 seq_printf(m, "ib ring tests failed (%d).\n", r);
981 else
982 seq_printf(m, "ib ring tests passed.\n");
983
984 /* go on the scheduler */
985 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
986 struct amdgpu_ring *ring = adev->rings[i];
987
988 if (!ring || !ring->sched.thread)
989 continue;
990 kthread_unpark(ring->sched.thread);
991 }
992
993 mutex_unlock(&adev->lock_reset);
994
995 pm_runtime_mark_last_busy(dev->dev);
996 pm_runtime_put_autosuspend(dev->dev);
997
998 return 0;
999 }
1000
amdgpu_debugfs_get_vbios_dump(struct seq_file * m,void * data)1001 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
1002 {
1003 struct drm_info_node *node = (struct drm_info_node *) m->private;
1004 struct drm_device *dev = node->minor->dev;
1005 struct amdgpu_device *adev = dev->dev_private;
1006
1007 seq_write(m, adev->bios, adev->bios_size);
1008 return 0;
1009 }
1010
amdgpu_debugfs_evict_vram(struct seq_file * m,void * data)1011 static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
1012 {
1013 struct drm_info_node *node = (struct drm_info_node *)m->private;
1014 struct drm_device *dev = node->minor->dev;
1015 struct amdgpu_device *adev = dev->dev_private;
1016 int r;
1017
1018 r = pm_runtime_get_sync(dev->dev);
1019 if (r < 0)
1020 return r;
1021
1022 seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
1023
1024 pm_runtime_mark_last_busy(dev->dev);
1025 pm_runtime_put_autosuspend(dev->dev);
1026
1027 return 0;
1028 }
1029
amdgpu_debugfs_evict_gtt(struct seq_file * m,void * data)1030 static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data)
1031 {
1032 struct drm_info_node *node = (struct drm_info_node *)m->private;
1033 struct drm_device *dev = node->minor->dev;
1034 struct amdgpu_device *adev = dev->dev_private;
1035 int r;
1036
1037 r = pm_runtime_get_sync(dev->dev);
1038 if (r < 0)
1039 return r;
1040
1041 seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
1042
1043 pm_runtime_mark_last_busy(dev->dev);
1044 pm_runtime_put_autosuspend(dev->dev);
1045
1046 return 0;
1047 }
1048
1049 static const struct drm_info_list amdgpu_debugfs_list[] = {
1050 {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
1051 {"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
1052 {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram},
1053 {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
1054 };
1055
amdgpu_ib_preempt_fences_swap(struct amdgpu_ring * ring,struct dma_fence ** fences)1056 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1057 struct dma_fence **fences)
1058 {
1059 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1060 uint32_t sync_seq, last_seq;
1061
1062 last_seq = atomic_read(&ring->fence_drv.last_seq);
1063 sync_seq = ring->fence_drv.sync_seq;
1064
1065 last_seq &= drv->num_fences_mask;
1066 sync_seq &= drv->num_fences_mask;
1067
1068 do {
1069 struct dma_fence *fence, **ptr;
1070
1071 ++last_seq;
1072 last_seq &= drv->num_fences_mask;
1073 ptr = &drv->fences[last_seq];
1074
1075 fence = rcu_dereference_protected(*ptr, 1);
1076 RCU_INIT_POINTER(*ptr, NULL);
1077
1078 if (!fence)
1079 continue;
1080
1081 fences[last_seq] = fence;
1082
1083 } while (last_seq != sync_seq);
1084 }
1085
amdgpu_ib_preempt_signal_fences(struct dma_fence ** fences,int length)1086 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1087 int length)
1088 {
1089 int i;
1090 struct dma_fence *fence;
1091
1092 for (i = 0; i < length; i++) {
1093 fence = fences[i];
1094 if (!fence)
1095 continue;
1096 dma_fence_signal(fence);
1097 dma_fence_put(fence);
1098 }
1099 }
1100
amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler * sched)1101 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1102 {
1103 struct drm_sched_job *s_job;
1104 struct dma_fence *fence;
1105
1106 spin_lock(&sched->job_list_lock);
1107 list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1108 fence = sched->ops->run_job(s_job);
1109 dma_fence_put(fence);
1110 }
1111 spin_unlock(&sched->job_list_lock);
1112 }
1113
amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring * ring)1114 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1115 {
1116 struct amdgpu_job *job;
1117 struct drm_sched_job *s_job;
1118 uint32_t preempt_seq;
1119 struct dma_fence *fence, **ptr;
1120 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1121 struct drm_gpu_scheduler *sched = &ring->sched;
1122
1123 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1124 return;
1125
1126 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1127 if (preempt_seq <= atomic_read(&drv->last_seq))
1128 return;
1129
1130 preempt_seq &= drv->num_fences_mask;
1131 ptr = &drv->fences[preempt_seq];
1132 fence = rcu_dereference_protected(*ptr, 1);
1133
1134 spin_lock(&sched->job_list_lock);
1135 list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1136 job = to_amdgpu_job(s_job);
1137 if (job->fence == fence)
1138 /* mark the job as preempted */
1139 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1140 }
1141 spin_unlock(&sched->job_list_lock);
1142 }
1143
amdgpu_debugfs_ib_preempt(void * data,u64 val)1144 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1145 {
1146 int r, resched, length;
1147 struct amdgpu_ring *ring;
1148 struct dma_fence **fences = NULL;
1149 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1150
1151 if (val >= AMDGPU_MAX_RINGS)
1152 return -EINVAL;
1153
1154 ring = adev->rings[val];
1155
1156 if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1157 return -EINVAL;
1158
1159 /* the last preemption failed */
1160 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1161 return -EBUSY;
1162
1163 length = ring->fence_drv.num_fences_mask + 1;
1164 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1165 if (!fences)
1166 return -ENOMEM;
1167
1168 /* Avoid accidently unparking the sched thread during GPU reset */
1169 mutex_lock(&adev->lock_reset);
1170
1171 /* stop the scheduler */
1172 kthread_park(ring->sched.thread);
1173
1174 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1175
1176 /* preempt the IB */
1177 r = amdgpu_ring_preempt_ib(ring);
1178 if (r) {
1179 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1180 goto failure;
1181 }
1182
1183 amdgpu_fence_process(ring);
1184
1185 if (atomic_read(&ring->fence_drv.last_seq) !=
1186 ring->fence_drv.sync_seq) {
1187 DRM_INFO("ring %d was preempted\n", ring->idx);
1188
1189 amdgpu_ib_preempt_mark_partial_job(ring);
1190
1191 /* swap out the old fences */
1192 amdgpu_ib_preempt_fences_swap(ring, fences);
1193
1194 amdgpu_fence_driver_force_completion(ring);
1195
1196 /* resubmit unfinished jobs */
1197 amdgpu_ib_preempt_job_recovery(&ring->sched);
1198
1199 /* wait for jobs finished */
1200 amdgpu_fence_wait_empty(ring);
1201
1202 /* signal the old fences */
1203 amdgpu_ib_preempt_signal_fences(fences, length);
1204 }
1205
1206 failure:
1207 /* restart the scheduler */
1208 kthread_unpark(ring->sched.thread);
1209
1210 mutex_unlock(&adev->lock_reset);
1211
1212 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1213
1214 kfree(fences);
1215
1216 return 0;
1217 }
1218
1219 DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
1220 amdgpu_debugfs_ib_preempt, "%llu\n");
1221
amdgpu_debugfs_init(struct amdgpu_device * adev)1222 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1223 {
1224 adev->debugfs_preempt =
1225 debugfs_create_file("amdgpu_preempt_ib", 0600,
1226 adev->ddev->primary->debugfs_root, adev,
1227 &fops_ib_preempt);
1228 if (!(adev->debugfs_preempt)) {
1229 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1230 return -EIO;
1231 }
1232
1233 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
1234 ARRAY_SIZE(amdgpu_debugfs_list));
1235 }
1236
amdgpu_debugfs_preempt_cleanup(struct amdgpu_device * adev)1237 void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev)
1238 {
1239 debugfs_remove(adev->debugfs_preempt);
1240 }
1241
1242 #else
amdgpu_debugfs_init(struct amdgpu_device * adev)1243 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1244 {
1245 return 0;
1246 }
amdgpu_debugfs_preempt_cleanup(struct amdgpu_device * adev)1247 void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev) { }
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)1248 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1249 {
1250 return 0;
1251 }
amdgpu_debugfs_regs_cleanup(struct amdgpu_device * adev)1252 void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
1253 #endif
1254