1 /* $NetBSD: amdgpu_cik.c,v 1.7 2023/09/30 10:46:45 mrg Exp $ */
2
3 /*
4 * Copyright 2012 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Alex Deucher
25 */
26 #include <sys/cdefs.h>
27 __KERNEL_RCSID(0, "$NetBSD: amdgpu_cik.c,v 1.7 2023/09/30 10:46:45 mrg Exp $");
28
29 #include <linux/firmware.h>
30 #include <linux/slab.h>
31 #include <linux/module.h>
32 #include <linux/pci.h>
33
34 #include "amdgpu.h"
35 #include "amdgpu_atombios.h"
36 #include "amdgpu_ih.h"
37 #include "amdgpu_uvd.h"
38 #include "amdgpu_vce.h"
39 #include "cikd.h"
40 #include "atom.h"
41 #include "amd_pcie.h"
42
43 #include "cik.h"
44 #include "gmc_v7_0.h"
45 #include "cik_ih.h"
46 #include "dce_v8_0.h"
47 #include "gfx_v7_0.h"
48 #include "cik_sdma.h"
49 #include "uvd_v4_2.h"
50 #include "vce_v2_0.h"
51 #include "cik_dpm.h"
52
53 #include "uvd/uvd_4_2_d.h"
54
55 #include "smu/smu_7_0_1_d.h"
56 #include "smu/smu_7_0_1_sh_mask.h"
57
58 #include "dce/dce_8_0_d.h"
59 #include "dce/dce_8_0_sh_mask.h"
60
61 #include "bif/bif_4_1_d.h"
62 #include "bif/bif_4_1_sh_mask.h"
63
64 #include "gca/gfx_7_2_d.h"
65 #include "gca/gfx_7_2_enum.h"
66 #include "gca/gfx_7_2_sh_mask.h"
67
68 #include "gmc/gmc_7_1_d.h"
69 #include "gmc/gmc_7_1_sh_mask.h"
70
71 #include "oss/oss_2_0_d.h"
72 #include "oss/oss_2_0_sh_mask.h"
73
74 #include "amdgpu_dm.h"
75 #include "amdgpu_amdkfd.h"
76 #include "dce_virtual.h"
77
78 #include <linux/nbsd-namespace.h>
79
80 /*
81 * Indirect registers accessor
82 */
cik_pcie_rreg(struct amdgpu_device * adev,u32 reg)83 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
84 {
85 unsigned long flags;
86 u32 r;
87
88 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
89 WREG32(mmPCIE_INDEX, reg);
90 (void)RREG32(mmPCIE_INDEX);
91 r = RREG32(mmPCIE_DATA);
92 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
93 return r;
94 }
95
cik_pcie_wreg(struct amdgpu_device * adev,u32 reg,u32 v)96 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
97 {
98 unsigned long flags;
99
100 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
101 WREG32(mmPCIE_INDEX, reg);
102 (void)RREG32(mmPCIE_INDEX);
103 WREG32(mmPCIE_DATA, v);
104 (void)RREG32(mmPCIE_DATA);
105 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
106 }
107
cik_smc_rreg(struct amdgpu_device * adev,u32 reg)108 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
109 {
110 unsigned long flags;
111 u32 r;
112
113 spin_lock_irqsave(&adev->smc_idx_lock, flags);
114 WREG32(mmSMC_IND_INDEX_0, (reg));
115 r = RREG32(mmSMC_IND_DATA_0);
116 spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
117 return r;
118 }
119
cik_smc_wreg(struct amdgpu_device * adev,u32 reg,u32 v)120 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
121 {
122 unsigned long flags;
123
124 spin_lock_irqsave(&adev->smc_idx_lock, flags);
125 WREG32(mmSMC_IND_INDEX_0, (reg));
126 WREG32(mmSMC_IND_DATA_0, (v));
127 spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
128 }
129
cik_uvd_ctx_rreg(struct amdgpu_device * adev,u32 reg)130 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
131 {
132 unsigned long flags;
133 u32 r;
134
135 spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
136 WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
137 r = RREG32(mmUVD_CTX_DATA);
138 spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
139 return r;
140 }
141
cik_uvd_ctx_wreg(struct amdgpu_device * adev,u32 reg,u32 v)142 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
143 {
144 unsigned long flags;
145
146 spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
147 WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
148 WREG32(mmUVD_CTX_DATA, (v));
149 spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
150 }
151
cik_didt_rreg(struct amdgpu_device * adev,u32 reg)152 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
153 {
154 unsigned long flags;
155 u32 r;
156
157 spin_lock_irqsave(&adev->didt_idx_lock, flags);
158 WREG32(mmDIDT_IND_INDEX, (reg));
159 r = RREG32(mmDIDT_IND_DATA);
160 spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
161 return r;
162 }
163
cik_didt_wreg(struct amdgpu_device * adev,u32 reg,u32 v)164 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
165 {
166 unsigned long flags;
167
168 spin_lock_irqsave(&adev->didt_idx_lock, flags);
169 WREG32(mmDIDT_IND_INDEX, (reg));
170 WREG32(mmDIDT_IND_DATA, (v));
171 spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
172 }
173
174 static const u32 bonaire_golden_spm_registers[] =
175 {
176 0xc200, 0xe0ffffff, 0xe0000000
177 };
178
179 static const u32 bonaire_golden_common_registers[] =
180 {
181 0x31dc, 0xffffffff, 0x00000800,
182 0x31dd, 0xffffffff, 0x00000800,
183 0x31e6, 0xffffffff, 0x00007fbf,
184 0x31e7, 0xffffffff, 0x00007faf
185 };
186
187 static const u32 bonaire_golden_registers[] =
188 {
189 0xcd5, 0x00000333, 0x00000333,
190 0xcd4, 0x000c0fc0, 0x00040200,
191 0x2684, 0x00010000, 0x00058208,
192 0xf000, 0xffff1fff, 0x00140000,
193 0xf080, 0xfdfc0fff, 0x00000100,
194 0xf08d, 0x40000000, 0x40000200,
195 0x260c, 0xffffffff, 0x00000000,
196 0x260d, 0xf00fffff, 0x00000400,
197 0x260e, 0x0002021c, 0x00020200,
198 0x31e, 0x00000080, 0x00000000,
199 0x16ec, 0x000000f0, 0x00000070,
200 0x16f0, 0xf0311fff, 0x80300000,
201 0x263e, 0x73773777, 0x12010001,
202 0xd43, 0x00810000, 0x408af000,
203 0x1c0c, 0x31000111, 0x00000011,
204 0xbd2, 0x73773777, 0x12010001,
205 0x883, 0x00007fb6, 0x0021a1b1,
206 0x884, 0x00007fb6, 0x002021b1,
207 0x860, 0x00007fb6, 0x00002191,
208 0x886, 0x00007fb6, 0x002121b1,
209 0x887, 0x00007fb6, 0x002021b1,
210 0x877, 0x00007fb6, 0x00002191,
211 0x878, 0x00007fb6, 0x00002191,
212 0xd8a, 0x0000003f, 0x0000000a,
213 0xd8b, 0x0000003f, 0x0000000a,
214 0xab9, 0x00073ffe, 0x000022a2,
215 0x903, 0x000007ff, 0x00000000,
216 0x2285, 0xf000003f, 0x00000007,
217 0x22fc, 0x00002001, 0x00000001,
218 0x22c9, 0xffffffff, 0x00ffffff,
219 0xc281, 0x0000ff0f, 0x00000000,
220 0xa293, 0x07ffffff, 0x06000000,
221 0x136, 0x00000fff, 0x00000100,
222 0xf9e, 0x00000001, 0x00000002,
223 0x2440, 0x03000000, 0x0362c688,
224 0x2300, 0x000000ff, 0x00000001,
225 0x390, 0x00001fff, 0x00001fff,
226 0x2418, 0x0000007f, 0x00000020,
227 0x2542, 0x00010000, 0x00010000,
228 0x2b05, 0x000003ff, 0x000000f3,
229 0x2b03, 0xffffffff, 0x00001032
230 };
231
232 static const u32 bonaire_mgcg_cgcg_init[] =
233 {
234 0x3108, 0xffffffff, 0xfffffffc,
235 0xc200, 0xffffffff, 0xe0000000,
236 0xf0a8, 0xffffffff, 0x00000100,
237 0xf082, 0xffffffff, 0x00000100,
238 0xf0b0, 0xffffffff, 0xc0000100,
239 0xf0b2, 0xffffffff, 0xc0000100,
240 0xf0b1, 0xffffffff, 0xc0000100,
241 0x1579, 0xffffffff, 0x00600100,
242 0xf0a0, 0xffffffff, 0x00000100,
243 0xf085, 0xffffffff, 0x06000100,
244 0xf088, 0xffffffff, 0x00000100,
245 0xf086, 0xffffffff, 0x06000100,
246 0xf081, 0xffffffff, 0x00000100,
247 0xf0b8, 0xffffffff, 0x00000100,
248 0xf089, 0xffffffff, 0x00000100,
249 0xf080, 0xffffffff, 0x00000100,
250 0xf08c, 0xffffffff, 0x00000100,
251 0xf08d, 0xffffffff, 0x00000100,
252 0xf094, 0xffffffff, 0x00000100,
253 0xf095, 0xffffffff, 0x00000100,
254 0xf096, 0xffffffff, 0x00000100,
255 0xf097, 0xffffffff, 0x00000100,
256 0xf098, 0xffffffff, 0x00000100,
257 0xf09f, 0xffffffff, 0x00000100,
258 0xf09e, 0xffffffff, 0x00000100,
259 0xf084, 0xffffffff, 0x06000100,
260 0xf0a4, 0xffffffff, 0x00000100,
261 0xf09d, 0xffffffff, 0x00000100,
262 0xf0ad, 0xffffffff, 0x00000100,
263 0xf0ac, 0xffffffff, 0x00000100,
264 0xf09c, 0xffffffff, 0x00000100,
265 0xc200, 0xffffffff, 0xe0000000,
266 0xf008, 0xffffffff, 0x00010000,
267 0xf009, 0xffffffff, 0x00030002,
268 0xf00a, 0xffffffff, 0x00040007,
269 0xf00b, 0xffffffff, 0x00060005,
270 0xf00c, 0xffffffff, 0x00090008,
271 0xf00d, 0xffffffff, 0x00010000,
272 0xf00e, 0xffffffff, 0x00030002,
273 0xf00f, 0xffffffff, 0x00040007,
274 0xf010, 0xffffffff, 0x00060005,
275 0xf011, 0xffffffff, 0x00090008,
276 0xf012, 0xffffffff, 0x00010000,
277 0xf013, 0xffffffff, 0x00030002,
278 0xf014, 0xffffffff, 0x00040007,
279 0xf015, 0xffffffff, 0x00060005,
280 0xf016, 0xffffffff, 0x00090008,
281 0xf017, 0xffffffff, 0x00010000,
282 0xf018, 0xffffffff, 0x00030002,
283 0xf019, 0xffffffff, 0x00040007,
284 0xf01a, 0xffffffff, 0x00060005,
285 0xf01b, 0xffffffff, 0x00090008,
286 0xf01c, 0xffffffff, 0x00010000,
287 0xf01d, 0xffffffff, 0x00030002,
288 0xf01e, 0xffffffff, 0x00040007,
289 0xf01f, 0xffffffff, 0x00060005,
290 0xf020, 0xffffffff, 0x00090008,
291 0xf021, 0xffffffff, 0x00010000,
292 0xf022, 0xffffffff, 0x00030002,
293 0xf023, 0xffffffff, 0x00040007,
294 0xf024, 0xffffffff, 0x00060005,
295 0xf025, 0xffffffff, 0x00090008,
296 0xf026, 0xffffffff, 0x00010000,
297 0xf027, 0xffffffff, 0x00030002,
298 0xf028, 0xffffffff, 0x00040007,
299 0xf029, 0xffffffff, 0x00060005,
300 0xf02a, 0xffffffff, 0x00090008,
301 0xf000, 0xffffffff, 0x96e00200,
302 0x21c2, 0xffffffff, 0x00900100,
303 0x3109, 0xffffffff, 0x0020003f,
304 0xe, 0xffffffff, 0x0140001c,
305 0xf, 0x000f0000, 0x000f0000,
306 0x88, 0xffffffff, 0xc060000c,
307 0x89, 0xc0000fff, 0x00000100,
308 0x3e4, 0xffffffff, 0x00000100,
309 0x3e6, 0x00000101, 0x00000000,
310 0x82a, 0xffffffff, 0x00000104,
311 0x1579, 0xff000fff, 0x00000100,
312 0xc33, 0xc0000fff, 0x00000104,
313 0x3079, 0x00000001, 0x00000001,
314 0x3403, 0xff000ff0, 0x00000100,
315 0x3603, 0xff000ff0, 0x00000100
316 };
317
318 static const u32 spectre_golden_spm_registers[] =
319 {
320 0xc200, 0xe0ffffff, 0xe0000000
321 };
322
323 static const u32 spectre_golden_common_registers[] =
324 {
325 0x31dc, 0xffffffff, 0x00000800,
326 0x31dd, 0xffffffff, 0x00000800,
327 0x31e6, 0xffffffff, 0x00007fbf,
328 0x31e7, 0xffffffff, 0x00007faf
329 };
330
331 static const u32 spectre_golden_registers[] =
332 {
333 0xf000, 0xffff1fff, 0x96940200,
334 0xf003, 0xffff0001, 0xff000000,
335 0xf080, 0xfffc0fff, 0x00000100,
336 0x1bb6, 0x00010101, 0x00010000,
337 0x260d, 0xf00fffff, 0x00000400,
338 0x260e, 0xfffffffc, 0x00020200,
339 0x16ec, 0x000000f0, 0x00000070,
340 0x16f0, 0xf0311fff, 0x80300000,
341 0x263e, 0x73773777, 0x12010001,
342 0x26df, 0x00ff0000, 0x00fc0000,
343 0xbd2, 0x73773777, 0x12010001,
344 0x2285, 0xf000003f, 0x00000007,
345 0x22c9, 0xffffffff, 0x00ffffff,
346 0xa0d4, 0x3f3f3fff, 0x00000082,
347 0xa0d5, 0x0000003f, 0x00000000,
348 0xf9e, 0x00000001, 0x00000002,
349 0x244f, 0xffff03df, 0x00000004,
350 0x31da, 0x00000008, 0x00000008,
351 0x2300, 0x000008ff, 0x00000800,
352 0x2542, 0x00010000, 0x00010000,
353 0x2b03, 0xffffffff, 0x54763210,
354 0x853e, 0x01ff01ff, 0x00000002,
355 0x8526, 0x007ff800, 0x00200000,
356 0x8057, 0xffffffff, 0x00000f40,
357 0xc24d, 0xffffffff, 0x00000001
358 };
359
360 static const u32 spectre_mgcg_cgcg_init[] =
361 {
362 0x3108, 0xffffffff, 0xfffffffc,
363 0xc200, 0xffffffff, 0xe0000000,
364 0xf0a8, 0xffffffff, 0x00000100,
365 0xf082, 0xffffffff, 0x00000100,
366 0xf0b0, 0xffffffff, 0x00000100,
367 0xf0b2, 0xffffffff, 0x00000100,
368 0xf0b1, 0xffffffff, 0x00000100,
369 0x1579, 0xffffffff, 0x00600100,
370 0xf0a0, 0xffffffff, 0x00000100,
371 0xf085, 0xffffffff, 0x06000100,
372 0xf088, 0xffffffff, 0x00000100,
373 0xf086, 0xffffffff, 0x06000100,
374 0xf081, 0xffffffff, 0x00000100,
375 0xf0b8, 0xffffffff, 0x00000100,
376 0xf089, 0xffffffff, 0x00000100,
377 0xf080, 0xffffffff, 0x00000100,
378 0xf08c, 0xffffffff, 0x00000100,
379 0xf08d, 0xffffffff, 0x00000100,
380 0xf094, 0xffffffff, 0x00000100,
381 0xf095, 0xffffffff, 0x00000100,
382 0xf096, 0xffffffff, 0x00000100,
383 0xf097, 0xffffffff, 0x00000100,
384 0xf098, 0xffffffff, 0x00000100,
385 0xf09f, 0xffffffff, 0x00000100,
386 0xf09e, 0xffffffff, 0x00000100,
387 0xf084, 0xffffffff, 0x06000100,
388 0xf0a4, 0xffffffff, 0x00000100,
389 0xf09d, 0xffffffff, 0x00000100,
390 0xf0ad, 0xffffffff, 0x00000100,
391 0xf0ac, 0xffffffff, 0x00000100,
392 0xf09c, 0xffffffff, 0x00000100,
393 0xc200, 0xffffffff, 0xe0000000,
394 0xf008, 0xffffffff, 0x00010000,
395 0xf009, 0xffffffff, 0x00030002,
396 0xf00a, 0xffffffff, 0x00040007,
397 0xf00b, 0xffffffff, 0x00060005,
398 0xf00c, 0xffffffff, 0x00090008,
399 0xf00d, 0xffffffff, 0x00010000,
400 0xf00e, 0xffffffff, 0x00030002,
401 0xf00f, 0xffffffff, 0x00040007,
402 0xf010, 0xffffffff, 0x00060005,
403 0xf011, 0xffffffff, 0x00090008,
404 0xf012, 0xffffffff, 0x00010000,
405 0xf013, 0xffffffff, 0x00030002,
406 0xf014, 0xffffffff, 0x00040007,
407 0xf015, 0xffffffff, 0x00060005,
408 0xf016, 0xffffffff, 0x00090008,
409 0xf017, 0xffffffff, 0x00010000,
410 0xf018, 0xffffffff, 0x00030002,
411 0xf019, 0xffffffff, 0x00040007,
412 0xf01a, 0xffffffff, 0x00060005,
413 0xf01b, 0xffffffff, 0x00090008,
414 0xf01c, 0xffffffff, 0x00010000,
415 0xf01d, 0xffffffff, 0x00030002,
416 0xf01e, 0xffffffff, 0x00040007,
417 0xf01f, 0xffffffff, 0x00060005,
418 0xf020, 0xffffffff, 0x00090008,
419 0xf021, 0xffffffff, 0x00010000,
420 0xf022, 0xffffffff, 0x00030002,
421 0xf023, 0xffffffff, 0x00040007,
422 0xf024, 0xffffffff, 0x00060005,
423 0xf025, 0xffffffff, 0x00090008,
424 0xf026, 0xffffffff, 0x00010000,
425 0xf027, 0xffffffff, 0x00030002,
426 0xf028, 0xffffffff, 0x00040007,
427 0xf029, 0xffffffff, 0x00060005,
428 0xf02a, 0xffffffff, 0x00090008,
429 0xf02b, 0xffffffff, 0x00010000,
430 0xf02c, 0xffffffff, 0x00030002,
431 0xf02d, 0xffffffff, 0x00040007,
432 0xf02e, 0xffffffff, 0x00060005,
433 0xf02f, 0xffffffff, 0x00090008,
434 0xf000, 0xffffffff, 0x96e00200,
435 0x21c2, 0xffffffff, 0x00900100,
436 0x3109, 0xffffffff, 0x0020003f,
437 0xe, 0xffffffff, 0x0140001c,
438 0xf, 0x000f0000, 0x000f0000,
439 0x88, 0xffffffff, 0xc060000c,
440 0x89, 0xc0000fff, 0x00000100,
441 0x3e4, 0xffffffff, 0x00000100,
442 0x3e6, 0x00000101, 0x00000000,
443 0x82a, 0xffffffff, 0x00000104,
444 0x1579, 0xff000fff, 0x00000100,
445 0xc33, 0xc0000fff, 0x00000104,
446 0x3079, 0x00000001, 0x00000001,
447 0x3403, 0xff000ff0, 0x00000100,
448 0x3603, 0xff000ff0, 0x00000100
449 };
450
451 static const u32 kalindi_golden_spm_registers[] =
452 {
453 0xc200, 0xe0ffffff, 0xe0000000
454 };
455
456 static const u32 kalindi_golden_common_registers[] =
457 {
458 0x31dc, 0xffffffff, 0x00000800,
459 0x31dd, 0xffffffff, 0x00000800,
460 0x31e6, 0xffffffff, 0x00007fbf,
461 0x31e7, 0xffffffff, 0x00007faf
462 };
463
464 static const u32 kalindi_golden_registers[] =
465 {
466 0xf000, 0xffffdfff, 0x6e944040,
467 0x1579, 0xff607fff, 0xfc000100,
468 0xf088, 0xff000fff, 0x00000100,
469 0xf089, 0xff000fff, 0x00000100,
470 0xf080, 0xfffc0fff, 0x00000100,
471 0x1bb6, 0x00010101, 0x00010000,
472 0x260c, 0xffffffff, 0x00000000,
473 0x260d, 0xf00fffff, 0x00000400,
474 0x16ec, 0x000000f0, 0x00000070,
475 0x16f0, 0xf0311fff, 0x80300000,
476 0x263e, 0x73773777, 0x12010001,
477 0x263f, 0xffffffff, 0x00000010,
478 0x26df, 0x00ff0000, 0x00fc0000,
479 0x200c, 0x00001f0f, 0x0000100a,
480 0xbd2, 0x73773777, 0x12010001,
481 0x902, 0x000fffff, 0x000c007f,
482 0x2285, 0xf000003f, 0x00000007,
483 0x22c9, 0x3fff3fff, 0x00ffcfff,
484 0xc281, 0x0000ff0f, 0x00000000,
485 0xa293, 0x07ffffff, 0x06000000,
486 0x136, 0x00000fff, 0x00000100,
487 0xf9e, 0x00000001, 0x00000002,
488 0x31da, 0x00000008, 0x00000008,
489 0x2300, 0x000000ff, 0x00000003,
490 0x853e, 0x01ff01ff, 0x00000002,
491 0x8526, 0x007ff800, 0x00200000,
492 0x8057, 0xffffffff, 0x00000f40,
493 0x2231, 0x001f3ae3, 0x00000082,
494 0x2235, 0x0000001f, 0x00000010,
495 0xc24d, 0xffffffff, 0x00000000
496 };
497
498 static const u32 kalindi_mgcg_cgcg_init[] =
499 {
500 0x3108, 0xffffffff, 0xfffffffc,
501 0xc200, 0xffffffff, 0xe0000000,
502 0xf0a8, 0xffffffff, 0x00000100,
503 0xf082, 0xffffffff, 0x00000100,
504 0xf0b0, 0xffffffff, 0x00000100,
505 0xf0b2, 0xffffffff, 0x00000100,
506 0xf0b1, 0xffffffff, 0x00000100,
507 0x1579, 0xffffffff, 0x00600100,
508 0xf0a0, 0xffffffff, 0x00000100,
509 0xf085, 0xffffffff, 0x06000100,
510 0xf088, 0xffffffff, 0x00000100,
511 0xf086, 0xffffffff, 0x06000100,
512 0xf081, 0xffffffff, 0x00000100,
513 0xf0b8, 0xffffffff, 0x00000100,
514 0xf089, 0xffffffff, 0x00000100,
515 0xf080, 0xffffffff, 0x00000100,
516 0xf08c, 0xffffffff, 0x00000100,
517 0xf08d, 0xffffffff, 0x00000100,
518 0xf094, 0xffffffff, 0x00000100,
519 0xf095, 0xffffffff, 0x00000100,
520 0xf096, 0xffffffff, 0x00000100,
521 0xf097, 0xffffffff, 0x00000100,
522 0xf098, 0xffffffff, 0x00000100,
523 0xf09f, 0xffffffff, 0x00000100,
524 0xf09e, 0xffffffff, 0x00000100,
525 0xf084, 0xffffffff, 0x06000100,
526 0xf0a4, 0xffffffff, 0x00000100,
527 0xf09d, 0xffffffff, 0x00000100,
528 0xf0ad, 0xffffffff, 0x00000100,
529 0xf0ac, 0xffffffff, 0x00000100,
530 0xf09c, 0xffffffff, 0x00000100,
531 0xc200, 0xffffffff, 0xe0000000,
532 0xf008, 0xffffffff, 0x00010000,
533 0xf009, 0xffffffff, 0x00030002,
534 0xf00a, 0xffffffff, 0x00040007,
535 0xf00b, 0xffffffff, 0x00060005,
536 0xf00c, 0xffffffff, 0x00090008,
537 0xf00d, 0xffffffff, 0x00010000,
538 0xf00e, 0xffffffff, 0x00030002,
539 0xf00f, 0xffffffff, 0x00040007,
540 0xf010, 0xffffffff, 0x00060005,
541 0xf011, 0xffffffff, 0x00090008,
542 0xf000, 0xffffffff, 0x96e00200,
543 0x21c2, 0xffffffff, 0x00900100,
544 0x3109, 0xffffffff, 0x0020003f,
545 0xe, 0xffffffff, 0x0140001c,
546 0xf, 0x000f0000, 0x000f0000,
547 0x88, 0xffffffff, 0xc060000c,
548 0x89, 0xc0000fff, 0x00000100,
549 0x82a, 0xffffffff, 0x00000104,
550 0x1579, 0xff000fff, 0x00000100,
551 0xc33, 0xc0000fff, 0x00000104,
552 0x3079, 0x00000001, 0x00000001,
553 0x3403, 0xff000ff0, 0x00000100,
554 0x3603, 0xff000ff0, 0x00000100
555 };
556
557 static const u32 hawaii_golden_spm_registers[] =
558 {
559 0xc200, 0xe0ffffff, 0xe0000000
560 };
561
562 static const u32 hawaii_golden_common_registers[] =
563 {
564 0xc200, 0xffffffff, 0xe0000000,
565 0xa0d4, 0xffffffff, 0x3a00161a,
566 0xa0d5, 0xffffffff, 0x0000002e,
567 0x2684, 0xffffffff, 0x00018208,
568 0x263e, 0xffffffff, 0x12011003
569 };
570
571 static const u32 hawaii_golden_registers[] =
572 {
573 0xcd5, 0x00000333, 0x00000333,
574 0x2684, 0x00010000, 0x00058208,
575 0x260c, 0xffffffff, 0x00000000,
576 0x260d, 0xf00fffff, 0x00000400,
577 0x260e, 0x0002021c, 0x00020200,
578 0x31e, 0x00000080, 0x00000000,
579 0x16ec, 0x000000f0, 0x00000070,
580 0x16f0, 0xf0311fff, 0x80300000,
581 0xd43, 0x00810000, 0x408af000,
582 0x1c0c, 0x31000111, 0x00000011,
583 0xbd2, 0x73773777, 0x12010001,
584 0x848, 0x0000007f, 0x0000001b,
585 0x877, 0x00007fb6, 0x00002191,
586 0xd8a, 0x0000003f, 0x0000000a,
587 0xd8b, 0x0000003f, 0x0000000a,
588 0xab9, 0x00073ffe, 0x000022a2,
589 0x903, 0x000007ff, 0x00000000,
590 0x22fc, 0x00002001, 0x00000001,
591 0x22c9, 0xffffffff, 0x00ffffff,
592 0xc281, 0x0000ff0f, 0x00000000,
593 0xa293, 0x07ffffff, 0x06000000,
594 0xf9e, 0x00000001, 0x00000002,
595 0x31da, 0x00000008, 0x00000008,
596 0x31dc, 0x00000f00, 0x00000800,
597 0x31dd, 0x00000f00, 0x00000800,
598 0x31e6, 0x00ffffff, 0x00ff7fbf,
599 0x31e7, 0x00ffffff, 0x00ff7faf,
600 0x2300, 0x000000ff, 0x00000800,
601 0x390, 0x00001fff, 0x00001fff,
602 0x2418, 0x0000007f, 0x00000020,
603 0x2542, 0x00010000, 0x00010000,
604 0x2b80, 0x00100000, 0x000ff07c,
605 0x2b05, 0x000003ff, 0x0000000f,
606 0x2b04, 0xffffffff, 0x7564fdec,
607 0x2b03, 0xffffffff, 0x3120b9a8,
608 0x2b02, 0x20000000, 0x0f9c0000
609 };
610
611 static const u32 hawaii_mgcg_cgcg_init[] =
612 {
613 0x3108, 0xffffffff, 0xfffffffd,
614 0xc200, 0xffffffff, 0xe0000000,
615 0xf0a8, 0xffffffff, 0x00000100,
616 0xf082, 0xffffffff, 0x00000100,
617 0xf0b0, 0xffffffff, 0x00000100,
618 0xf0b2, 0xffffffff, 0x00000100,
619 0xf0b1, 0xffffffff, 0x00000100,
620 0x1579, 0xffffffff, 0x00200100,
621 0xf0a0, 0xffffffff, 0x00000100,
622 0xf085, 0xffffffff, 0x06000100,
623 0xf088, 0xffffffff, 0x00000100,
624 0xf086, 0xffffffff, 0x06000100,
625 0xf081, 0xffffffff, 0x00000100,
626 0xf0b8, 0xffffffff, 0x00000100,
627 0xf089, 0xffffffff, 0x00000100,
628 0xf080, 0xffffffff, 0x00000100,
629 0xf08c, 0xffffffff, 0x00000100,
630 0xf08d, 0xffffffff, 0x00000100,
631 0xf094, 0xffffffff, 0x00000100,
632 0xf095, 0xffffffff, 0x00000100,
633 0xf096, 0xffffffff, 0x00000100,
634 0xf097, 0xffffffff, 0x00000100,
635 0xf098, 0xffffffff, 0x00000100,
636 0xf09f, 0xffffffff, 0x00000100,
637 0xf09e, 0xffffffff, 0x00000100,
638 0xf084, 0xffffffff, 0x06000100,
639 0xf0a4, 0xffffffff, 0x00000100,
640 0xf09d, 0xffffffff, 0x00000100,
641 0xf0ad, 0xffffffff, 0x00000100,
642 0xf0ac, 0xffffffff, 0x00000100,
643 0xf09c, 0xffffffff, 0x00000100,
644 0xc200, 0xffffffff, 0xe0000000,
645 0xf008, 0xffffffff, 0x00010000,
646 0xf009, 0xffffffff, 0x00030002,
647 0xf00a, 0xffffffff, 0x00040007,
648 0xf00b, 0xffffffff, 0x00060005,
649 0xf00c, 0xffffffff, 0x00090008,
650 0xf00d, 0xffffffff, 0x00010000,
651 0xf00e, 0xffffffff, 0x00030002,
652 0xf00f, 0xffffffff, 0x00040007,
653 0xf010, 0xffffffff, 0x00060005,
654 0xf011, 0xffffffff, 0x00090008,
655 0xf012, 0xffffffff, 0x00010000,
656 0xf013, 0xffffffff, 0x00030002,
657 0xf014, 0xffffffff, 0x00040007,
658 0xf015, 0xffffffff, 0x00060005,
659 0xf016, 0xffffffff, 0x00090008,
660 0xf017, 0xffffffff, 0x00010000,
661 0xf018, 0xffffffff, 0x00030002,
662 0xf019, 0xffffffff, 0x00040007,
663 0xf01a, 0xffffffff, 0x00060005,
664 0xf01b, 0xffffffff, 0x00090008,
665 0xf01c, 0xffffffff, 0x00010000,
666 0xf01d, 0xffffffff, 0x00030002,
667 0xf01e, 0xffffffff, 0x00040007,
668 0xf01f, 0xffffffff, 0x00060005,
669 0xf020, 0xffffffff, 0x00090008,
670 0xf021, 0xffffffff, 0x00010000,
671 0xf022, 0xffffffff, 0x00030002,
672 0xf023, 0xffffffff, 0x00040007,
673 0xf024, 0xffffffff, 0x00060005,
674 0xf025, 0xffffffff, 0x00090008,
675 0xf026, 0xffffffff, 0x00010000,
676 0xf027, 0xffffffff, 0x00030002,
677 0xf028, 0xffffffff, 0x00040007,
678 0xf029, 0xffffffff, 0x00060005,
679 0xf02a, 0xffffffff, 0x00090008,
680 0xf02b, 0xffffffff, 0x00010000,
681 0xf02c, 0xffffffff, 0x00030002,
682 0xf02d, 0xffffffff, 0x00040007,
683 0xf02e, 0xffffffff, 0x00060005,
684 0xf02f, 0xffffffff, 0x00090008,
685 0xf030, 0xffffffff, 0x00010000,
686 0xf031, 0xffffffff, 0x00030002,
687 0xf032, 0xffffffff, 0x00040007,
688 0xf033, 0xffffffff, 0x00060005,
689 0xf034, 0xffffffff, 0x00090008,
690 0xf035, 0xffffffff, 0x00010000,
691 0xf036, 0xffffffff, 0x00030002,
692 0xf037, 0xffffffff, 0x00040007,
693 0xf038, 0xffffffff, 0x00060005,
694 0xf039, 0xffffffff, 0x00090008,
695 0xf03a, 0xffffffff, 0x00010000,
696 0xf03b, 0xffffffff, 0x00030002,
697 0xf03c, 0xffffffff, 0x00040007,
698 0xf03d, 0xffffffff, 0x00060005,
699 0xf03e, 0xffffffff, 0x00090008,
700 0x30c6, 0xffffffff, 0x00020200,
701 0xcd4, 0xffffffff, 0x00000200,
702 0x570, 0xffffffff, 0x00000400,
703 0x157a, 0xffffffff, 0x00000000,
704 0xbd4, 0xffffffff, 0x00000902,
705 0xf000, 0xffffffff, 0x96940200,
706 0x21c2, 0xffffffff, 0x00900100,
707 0x3109, 0xffffffff, 0x0020003f,
708 0xe, 0xffffffff, 0x0140001c,
709 0xf, 0x000f0000, 0x000f0000,
710 0x88, 0xffffffff, 0xc060000c,
711 0x89, 0xc0000fff, 0x00000100,
712 0x3e4, 0xffffffff, 0x00000100,
713 0x3e6, 0x00000101, 0x00000000,
714 0x82a, 0xffffffff, 0x00000104,
715 0x1579, 0xff000fff, 0x00000100,
716 0xc33, 0xc0000fff, 0x00000104,
717 0x3079, 0x00000001, 0x00000001,
718 0x3403, 0xff000ff0, 0x00000100,
719 0x3603, 0xff000ff0, 0x00000100
720 };
721
722 static const u32 godavari_golden_registers[] =
723 {
724 0x1579, 0xff607fff, 0xfc000100,
725 0x1bb6, 0x00010101, 0x00010000,
726 0x260c, 0xffffffff, 0x00000000,
727 0x260c0, 0xf00fffff, 0x00000400,
728 0x184c, 0xffffffff, 0x00010000,
729 0x16ec, 0x000000f0, 0x00000070,
730 0x16f0, 0xf0311fff, 0x80300000,
731 0x263e, 0x73773777, 0x12010001,
732 0x263f, 0xffffffff, 0x00000010,
733 0x200c, 0x00001f0f, 0x0000100a,
734 0xbd2, 0x73773777, 0x12010001,
735 0x902, 0x000fffff, 0x000c007f,
736 0x2285, 0xf000003f, 0x00000007,
737 0x22c9, 0xffffffff, 0x00ff0fff,
738 0xc281, 0x0000ff0f, 0x00000000,
739 0xa293, 0x07ffffff, 0x06000000,
740 0x136, 0x00000fff, 0x00000100,
741 0x3405, 0x00010000, 0x00810001,
742 0x3605, 0x00010000, 0x00810001,
743 0xf9e, 0x00000001, 0x00000002,
744 0x31da, 0x00000008, 0x00000008,
745 0x31dc, 0x00000f00, 0x00000800,
746 0x31dd, 0x00000f00, 0x00000800,
747 0x31e6, 0x00ffffff, 0x00ff7fbf,
748 0x31e7, 0x00ffffff, 0x00ff7faf,
749 0x2300, 0x000000ff, 0x00000001,
750 0x853e, 0x01ff01ff, 0x00000002,
751 0x8526, 0x007ff800, 0x00200000,
752 0x8057, 0xffffffff, 0x00000f40,
753 0x2231, 0x001f3ae3, 0x00000082,
754 0x2235, 0x0000001f, 0x00000010,
755 0xc24d, 0xffffffff, 0x00000000
756 };
757
cik_init_golden_registers(struct amdgpu_device * adev)758 static void cik_init_golden_registers(struct amdgpu_device *adev)
759 {
760 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
761 mutex_lock(&adev->grbm_idx_mutex);
762
763 switch (adev->asic_type) {
764 case CHIP_BONAIRE:
765 amdgpu_device_program_register_sequence(adev,
766 bonaire_mgcg_cgcg_init,
767 ARRAY_SIZE(bonaire_mgcg_cgcg_init));
768 amdgpu_device_program_register_sequence(adev,
769 bonaire_golden_registers,
770 ARRAY_SIZE(bonaire_golden_registers));
771 amdgpu_device_program_register_sequence(adev,
772 bonaire_golden_common_registers,
773 ARRAY_SIZE(bonaire_golden_common_registers));
774 amdgpu_device_program_register_sequence(adev,
775 bonaire_golden_spm_registers,
776 ARRAY_SIZE(bonaire_golden_spm_registers));
777 break;
778 case CHIP_KABINI:
779 amdgpu_device_program_register_sequence(adev,
780 kalindi_mgcg_cgcg_init,
781 ARRAY_SIZE(kalindi_mgcg_cgcg_init));
782 amdgpu_device_program_register_sequence(adev,
783 kalindi_golden_registers,
784 ARRAY_SIZE(kalindi_golden_registers));
785 amdgpu_device_program_register_sequence(adev,
786 kalindi_golden_common_registers,
787 ARRAY_SIZE(kalindi_golden_common_registers));
788 amdgpu_device_program_register_sequence(adev,
789 kalindi_golden_spm_registers,
790 ARRAY_SIZE(kalindi_golden_spm_registers));
791 break;
792 case CHIP_MULLINS:
793 amdgpu_device_program_register_sequence(adev,
794 kalindi_mgcg_cgcg_init,
795 ARRAY_SIZE(kalindi_mgcg_cgcg_init));
796 amdgpu_device_program_register_sequence(adev,
797 godavari_golden_registers,
798 ARRAY_SIZE(godavari_golden_registers));
799 amdgpu_device_program_register_sequence(adev,
800 kalindi_golden_common_registers,
801 ARRAY_SIZE(kalindi_golden_common_registers));
802 amdgpu_device_program_register_sequence(adev,
803 kalindi_golden_spm_registers,
804 ARRAY_SIZE(kalindi_golden_spm_registers));
805 break;
806 case CHIP_KAVERI:
807 amdgpu_device_program_register_sequence(adev,
808 spectre_mgcg_cgcg_init,
809 ARRAY_SIZE(spectre_mgcg_cgcg_init));
810 amdgpu_device_program_register_sequence(adev,
811 spectre_golden_registers,
812 ARRAY_SIZE(spectre_golden_registers));
813 amdgpu_device_program_register_sequence(adev,
814 spectre_golden_common_registers,
815 ARRAY_SIZE(spectre_golden_common_registers));
816 amdgpu_device_program_register_sequence(adev,
817 spectre_golden_spm_registers,
818 ARRAY_SIZE(spectre_golden_spm_registers));
819 break;
820 case CHIP_HAWAII:
821 amdgpu_device_program_register_sequence(adev,
822 hawaii_mgcg_cgcg_init,
823 ARRAY_SIZE(hawaii_mgcg_cgcg_init));
824 amdgpu_device_program_register_sequence(adev,
825 hawaii_golden_registers,
826 ARRAY_SIZE(hawaii_golden_registers));
827 amdgpu_device_program_register_sequence(adev,
828 hawaii_golden_common_registers,
829 ARRAY_SIZE(hawaii_golden_common_registers));
830 amdgpu_device_program_register_sequence(adev,
831 hawaii_golden_spm_registers,
832 ARRAY_SIZE(hawaii_golden_spm_registers));
833 break;
834 default:
835 break;
836 }
837 mutex_unlock(&adev->grbm_idx_mutex);
838 }
839
840 /**
841 * cik_get_xclk - get the xclk
842 *
843 * @adev: amdgpu_device pointer
844 *
845 * Returns the reference clock used by the gfx engine
846 * (CIK).
847 */
cik_get_xclk(struct amdgpu_device * adev)848 static u32 cik_get_xclk(struct amdgpu_device *adev)
849 {
850 u32 reference_clock = adev->clock.spll.reference_freq;
851
852 if (adev->flags & AMD_IS_APU) {
853 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
854 return reference_clock / 2;
855 } else {
856 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
857 return reference_clock / 4;
858 }
859 return reference_clock;
860 }
861
862 /**
863 * cik_srbm_select - select specific register instances
864 *
865 * @adev: amdgpu_device pointer
866 * @me: selected ME (micro engine)
867 * @pipe: pipe
868 * @queue: queue
869 * @vmid: VMID
870 *
871 * Switches the currently active registers instances. Some
872 * registers are instanced per VMID, others are instanced per
873 * me/pipe/queue combination.
874 */
cik_srbm_select(struct amdgpu_device * adev,u32 me,u32 pipe,u32 queue,u32 vmid)875 void cik_srbm_select(struct amdgpu_device *adev,
876 u32 me, u32 pipe, u32 queue, u32 vmid)
877 {
878 u32 srbm_gfx_cntl =
879 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
880 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
881 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
882 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
883 WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
884 }
885
cik_vga_set_state(struct amdgpu_device * adev,bool state)886 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
887 {
888 uint32_t tmp;
889
890 tmp = RREG32(mmCONFIG_CNTL);
891 if (!state)
892 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
893 else
894 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
895 WREG32(mmCONFIG_CNTL, tmp);
896 }
897
cik_read_disabled_bios(struct amdgpu_device * adev)898 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
899 {
900 u32 bus_cntl;
901 u32 d1vga_control = 0;
902 u32 d2vga_control = 0;
903 u32 vga_render_control = 0;
904 u32 rom_cntl;
905 bool r;
906
907 bus_cntl = RREG32(mmBUS_CNTL);
908 if (adev->mode_info.num_crtc) {
909 d1vga_control = RREG32(mmD1VGA_CONTROL);
910 d2vga_control = RREG32(mmD2VGA_CONTROL);
911 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
912 }
913 rom_cntl = RREG32_SMC(ixROM_CNTL);
914
915 /* enable the rom */
916 WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
917 if (adev->mode_info.num_crtc) {
918 /* Disable VGA mode */
919 WREG32(mmD1VGA_CONTROL,
920 (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
921 D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
922 WREG32(mmD2VGA_CONTROL,
923 (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
924 D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
925 WREG32(mmVGA_RENDER_CONTROL,
926 (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
927 }
928 WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
929
930 r = amdgpu_read_bios(adev);
931
932 /* restore regs */
933 WREG32(mmBUS_CNTL, bus_cntl);
934 if (adev->mode_info.num_crtc) {
935 WREG32(mmD1VGA_CONTROL, d1vga_control);
936 WREG32(mmD2VGA_CONTROL, d2vga_control);
937 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
938 }
939 WREG32_SMC(ixROM_CNTL, rom_cntl);
940 return r;
941 }
942
cik_read_bios_from_rom(struct amdgpu_device * adev,u8 * bios,u32 length_bytes)943 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
944 u8 *bios, u32 length_bytes)
945 {
946 u32 *dw_ptr;
947 unsigned long flags;
948 u32 i, length_dw;
949
950 if (bios == NULL)
951 return false;
952 if (length_bytes == 0)
953 return false;
954 /* APU vbios image is part of sbios image */
955 if (adev->flags & AMD_IS_APU)
956 return false;
957
958 dw_ptr = (u32 *)bios;
959 length_dw = ALIGN(length_bytes, 4) / 4;
960 /* take the smc lock since we are using the smc index */
961 spin_lock_irqsave(&adev->smc_idx_lock, flags);
962 /* set rom index to 0 */
963 WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
964 WREG32(mmSMC_IND_DATA_0, 0);
965 /* set index to data for continous read */
966 WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
967 for (i = 0; i < length_dw; i++)
968 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
969 spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
970
971 return true;
972 }
973
974 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
975 {mmGRBM_STATUS},
976 {mmGRBM_STATUS2},
977 {mmGRBM_STATUS_SE0},
978 {mmGRBM_STATUS_SE1},
979 {mmGRBM_STATUS_SE2},
980 {mmGRBM_STATUS_SE3},
981 {mmSRBM_STATUS},
982 {mmSRBM_STATUS2},
983 {mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
984 {mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
985 {mmCP_STAT},
986 {mmCP_STALLED_STAT1},
987 {mmCP_STALLED_STAT2},
988 {mmCP_STALLED_STAT3},
989 {mmCP_CPF_BUSY_STAT},
990 {mmCP_CPF_STALLED_STAT1},
991 {mmCP_CPF_STATUS},
992 {mmCP_CPC_BUSY_STAT},
993 {mmCP_CPC_STALLED_STAT1},
994 {mmCP_CPC_STATUS},
995 {mmGB_ADDR_CONFIG},
996 {mmMC_ARB_RAMCFG},
997 {mmGB_TILE_MODE0},
998 {mmGB_TILE_MODE1},
999 {mmGB_TILE_MODE2},
1000 {mmGB_TILE_MODE3},
1001 {mmGB_TILE_MODE4},
1002 {mmGB_TILE_MODE5},
1003 {mmGB_TILE_MODE6},
1004 {mmGB_TILE_MODE7},
1005 {mmGB_TILE_MODE8},
1006 {mmGB_TILE_MODE9},
1007 {mmGB_TILE_MODE10},
1008 {mmGB_TILE_MODE11},
1009 {mmGB_TILE_MODE12},
1010 {mmGB_TILE_MODE13},
1011 {mmGB_TILE_MODE14},
1012 {mmGB_TILE_MODE15},
1013 {mmGB_TILE_MODE16},
1014 {mmGB_TILE_MODE17},
1015 {mmGB_TILE_MODE18},
1016 {mmGB_TILE_MODE19},
1017 {mmGB_TILE_MODE20},
1018 {mmGB_TILE_MODE21},
1019 {mmGB_TILE_MODE22},
1020 {mmGB_TILE_MODE23},
1021 {mmGB_TILE_MODE24},
1022 {mmGB_TILE_MODE25},
1023 {mmGB_TILE_MODE26},
1024 {mmGB_TILE_MODE27},
1025 {mmGB_TILE_MODE28},
1026 {mmGB_TILE_MODE29},
1027 {mmGB_TILE_MODE30},
1028 {mmGB_TILE_MODE31},
1029 {mmGB_MACROTILE_MODE0},
1030 {mmGB_MACROTILE_MODE1},
1031 {mmGB_MACROTILE_MODE2},
1032 {mmGB_MACROTILE_MODE3},
1033 {mmGB_MACROTILE_MODE4},
1034 {mmGB_MACROTILE_MODE5},
1035 {mmGB_MACROTILE_MODE6},
1036 {mmGB_MACROTILE_MODE7},
1037 {mmGB_MACROTILE_MODE8},
1038 {mmGB_MACROTILE_MODE9},
1039 {mmGB_MACROTILE_MODE10},
1040 {mmGB_MACROTILE_MODE11},
1041 {mmGB_MACROTILE_MODE12},
1042 {mmGB_MACROTILE_MODE13},
1043 {mmGB_MACROTILE_MODE14},
1044 {mmGB_MACROTILE_MODE15},
1045 {mmCC_RB_BACKEND_DISABLE, true},
1046 {mmGC_USER_RB_BACKEND_DISABLE, true},
1047 {mmGB_BACKEND_MAP, false},
1048 {mmPA_SC_RASTER_CONFIG, true},
1049 {mmPA_SC_RASTER_CONFIG_1, true},
1050 };
1051
1052
cik_get_register_value(struct amdgpu_device * adev,bool indexed,u32 se_num,u32 sh_num,u32 reg_offset)1053 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1054 bool indexed, u32 se_num,
1055 u32 sh_num, u32 reg_offset)
1056 {
1057 if (indexed) {
1058 uint32_t val;
1059 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1060 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1061
1062 switch (reg_offset) {
1063 case mmCC_RB_BACKEND_DISABLE:
1064 return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1065 case mmGC_USER_RB_BACKEND_DISABLE:
1066 return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1067 case mmPA_SC_RASTER_CONFIG:
1068 return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1069 case mmPA_SC_RASTER_CONFIG_1:
1070 return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1071 }
1072
1073 mutex_lock(&adev->grbm_idx_mutex);
1074 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1075 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1076
1077 val = RREG32(reg_offset);
1078
1079 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1080 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1081 mutex_unlock(&adev->grbm_idx_mutex);
1082 return val;
1083 } else {
1084 unsigned idx;
1085
1086 switch (reg_offset) {
1087 case mmGB_ADDR_CONFIG:
1088 return adev->gfx.config.gb_addr_config;
1089 case mmMC_ARB_RAMCFG:
1090 return adev->gfx.config.mc_arb_ramcfg;
1091 case mmGB_TILE_MODE0:
1092 case mmGB_TILE_MODE1:
1093 case mmGB_TILE_MODE2:
1094 case mmGB_TILE_MODE3:
1095 case mmGB_TILE_MODE4:
1096 case mmGB_TILE_MODE5:
1097 case mmGB_TILE_MODE6:
1098 case mmGB_TILE_MODE7:
1099 case mmGB_TILE_MODE8:
1100 case mmGB_TILE_MODE9:
1101 case mmGB_TILE_MODE10:
1102 case mmGB_TILE_MODE11:
1103 case mmGB_TILE_MODE12:
1104 case mmGB_TILE_MODE13:
1105 case mmGB_TILE_MODE14:
1106 case mmGB_TILE_MODE15:
1107 case mmGB_TILE_MODE16:
1108 case mmGB_TILE_MODE17:
1109 case mmGB_TILE_MODE18:
1110 case mmGB_TILE_MODE19:
1111 case mmGB_TILE_MODE20:
1112 case mmGB_TILE_MODE21:
1113 case mmGB_TILE_MODE22:
1114 case mmGB_TILE_MODE23:
1115 case mmGB_TILE_MODE24:
1116 case mmGB_TILE_MODE25:
1117 case mmGB_TILE_MODE26:
1118 case mmGB_TILE_MODE27:
1119 case mmGB_TILE_MODE28:
1120 case mmGB_TILE_MODE29:
1121 case mmGB_TILE_MODE30:
1122 case mmGB_TILE_MODE31:
1123 idx = (reg_offset - mmGB_TILE_MODE0);
1124 return adev->gfx.config.tile_mode_array[idx];
1125 case mmGB_MACROTILE_MODE0:
1126 case mmGB_MACROTILE_MODE1:
1127 case mmGB_MACROTILE_MODE2:
1128 case mmGB_MACROTILE_MODE3:
1129 case mmGB_MACROTILE_MODE4:
1130 case mmGB_MACROTILE_MODE5:
1131 case mmGB_MACROTILE_MODE6:
1132 case mmGB_MACROTILE_MODE7:
1133 case mmGB_MACROTILE_MODE8:
1134 case mmGB_MACROTILE_MODE9:
1135 case mmGB_MACROTILE_MODE10:
1136 case mmGB_MACROTILE_MODE11:
1137 case mmGB_MACROTILE_MODE12:
1138 case mmGB_MACROTILE_MODE13:
1139 case mmGB_MACROTILE_MODE14:
1140 case mmGB_MACROTILE_MODE15:
1141 idx = (reg_offset - mmGB_MACROTILE_MODE0);
1142 return adev->gfx.config.macrotile_mode_array[idx];
1143 default:
1144 return RREG32(reg_offset);
1145 }
1146 }
1147 }
1148
cik_read_register(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 reg_offset,u32 * value)1149 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1150 u32 sh_num, u32 reg_offset, u32 *value)
1151 {
1152 uint32_t i;
1153
1154 *value = 0;
1155 for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1156 bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1157
1158 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1159 continue;
1160
1161 *value = cik_get_register_value(adev, indexed, se_num, sh_num,
1162 reg_offset);
1163 return 0;
1164 }
1165 return -EINVAL;
1166 }
1167
1168 struct kv_reset_save_regs {
1169 u32 gmcon_reng_execute;
1170 u32 gmcon_misc;
1171 u32 gmcon_misc3;
1172 };
1173
kv_save_regs_for_reset(struct amdgpu_device * adev,struct kv_reset_save_regs * save)1174 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1175 struct kv_reset_save_regs *save)
1176 {
1177 save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1178 save->gmcon_misc = RREG32(mmGMCON_MISC);
1179 save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1180
1181 WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1182 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1183 WREG32(mmGMCON_MISC, save->gmcon_misc &
1184 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1185 GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1186 }
1187
kv_restore_regs_for_reset(struct amdgpu_device * adev,struct kv_reset_save_regs * save)1188 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1189 struct kv_reset_save_regs *save)
1190 {
1191 int i;
1192
1193 WREG32(mmGMCON_PGFSM_WRITE, 0);
1194 WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1195
1196 for (i = 0; i < 5; i++)
1197 WREG32(mmGMCON_PGFSM_WRITE, 0);
1198
1199 WREG32(mmGMCON_PGFSM_WRITE, 0);
1200 WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1201
1202 for (i = 0; i < 5; i++)
1203 WREG32(mmGMCON_PGFSM_WRITE, 0);
1204
1205 WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1206 WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1207
1208 for (i = 0; i < 5; i++)
1209 WREG32(mmGMCON_PGFSM_WRITE, 0);
1210
1211 WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1212 WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1213
1214 for (i = 0; i < 5; i++)
1215 WREG32(mmGMCON_PGFSM_WRITE, 0);
1216
1217 WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1218 WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1219
1220 for (i = 0; i < 5; i++)
1221 WREG32(mmGMCON_PGFSM_WRITE, 0);
1222
1223 WREG32(mmGMCON_PGFSM_WRITE, 0);
1224 WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1225
1226 for (i = 0; i < 5; i++)
1227 WREG32(mmGMCON_PGFSM_WRITE, 0);
1228
1229 WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1230 WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1231
1232 for (i = 0; i < 5; i++)
1233 WREG32(mmGMCON_PGFSM_WRITE, 0);
1234
1235 WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1236 WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1237
1238 for (i = 0; i < 5; i++)
1239 WREG32(mmGMCON_PGFSM_WRITE, 0);
1240
1241 WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1242 WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1243
1244 for (i = 0; i < 5; i++)
1245 WREG32(mmGMCON_PGFSM_WRITE, 0);
1246
1247 WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1248 WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1249
1250 for (i = 0; i < 5; i++)
1251 WREG32(mmGMCON_PGFSM_WRITE, 0);
1252
1253 WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1254 WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1255
1256 WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1257 WREG32(mmGMCON_MISC, save->gmcon_misc);
1258 WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1259 }
1260
cik_gpu_pci_config_reset(struct amdgpu_device * adev)1261 static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1262 {
1263 struct kv_reset_save_regs kv_save = { 0 };
1264 u32 i;
1265 int r = -EINVAL;
1266
1267 dev_info(adev->dev, "GPU pci config reset\n");
1268
1269 if (adev->flags & AMD_IS_APU)
1270 kv_save_regs_for_reset(adev, &kv_save);
1271
1272 /* disable BM */
1273 pci_clear_master(adev->pdev);
1274 /* reset */
1275 amdgpu_device_pci_config_reset(adev);
1276
1277 udelay(100);
1278
1279 /* wait for asic to come out of reset */
1280 for (i = 0; i < adev->usec_timeout; i++) {
1281 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1282 /* enable BM */
1283 pci_set_master(adev->pdev);
1284 adev->has_hw_reset = true;
1285 r = 0;
1286 break;
1287 }
1288 udelay(1);
1289 }
1290
1291 /* does asic init need to be run first??? */
1292 if (adev->flags & AMD_IS_APU)
1293 kv_restore_regs_for_reset(adev, &kv_save);
1294
1295 return r;
1296 }
1297
1298 /**
1299 * cik_asic_pci_config_reset - soft reset GPU
1300 *
1301 * @adev: amdgpu_device pointer
1302 *
1303 * Use PCI Config method to reset the GPU.
1304 *
1305 * Returns 0 for success.
1306 */
cik_asic_pci_config_reset(struct amdgpu_device * adev)1307 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1308 {
1309 int r;
1310
1311 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1312
1313 r = cik_gpu_pci_config_reset(adev);
1314
1315 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1316
1317 return r;
1318 }
1319
cik_asic_supports_baco(struct amdgpu_device * adev)1320 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1321 {
1322 switch (adev->asic_type) {
1323 case CHIP_BONAIRE:
1324 case CHIP_HAWAII:
1325 return amdgpu_dpm_is_baco_supported(adev);
1326 default:
1327 return false;
1328 }
1329 }
1330
1331 static enum amd_reset_method
cik_asic_reset_method(struct amdgpu_device * adev)1332 cik_asic_reset_method(struct amdgpu_device *adev)
1333 {
1334 bool baco_reset;
1335
1336 switch (adev->asic_type) {
1337 case CHIP_BONAIRE:
1338 case CHIP_HAWAII:
1339 /* disable baco reset until it works */
1340 /* smu7_asic_get_baco_capability(adev, &baco_reset); */
1341 baco_reset = false;
1342 break;
1343 default:
1344 baco_reset = false;
1345 break;
1346 }
1347
1348 if (baco_reset)
1349 return AMD_RESET_METHOD_BACO;
1350 else
1351 return AMD_RESET_METHOD_LEGACY;
1352 }
1353
1354 /**
1355 * cik_asic_reset - soft reset GPU
1356 *
1357 * @adev: amdgpu_device pointer
1358 *
1359 * Look up which blocks are hung and attempt
1360 * to reset them.
1361 * Returns 0 for success.
1362 */
cik_asic_reset(struct amdgpu_device * adev)1363 static int cik_asic_reset(struct amdgpu_device *adev)
1364 {
1365 int r;
1366
1367 if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1368 if (!adev->in_suspend)
1369 amdgpu_inc_vram_lost(adev);
1370 r = amdgpu_dpm_baco_reset(adev);
1371 } else {
1372 r = cik_asic_pci_config_reset(adev);
1373 }
1374
1375 return r;
1376 }
1377
cik_get_config_memsize(struct amdgpu_device * adev)1378 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1379 {
1380 return RREG32(mmCONFIG_MEMSIZE);
1381 }
1382
cik_set_uvd_clock(struct amdgpu_device * adev,u32 clock,u32 cntl_reg,u32 status_reg)1383 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1384 u32 cntl_reg, u32 status_reg)
1385 {
1386 int r, i;
1387 struct atom_clock_dividers dividers;
1388 uint32_t tmp;
1389
1390 r = amdgpu_atombios_get_clock_dividers(adev,
1391 COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1392 clock, false, ÷rs);
1393 if (r)
1394 return r;
1395
1396 tmp = RREG32_SMC(cntl_reg);
1397 tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1398 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1399 tmp |= dividers.post_divider;
1400 WREG32_SMC(cntl_reg, tmp);
1401
1402 for (i = 0; i < 100; i++) {
1403 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1404 break;
1405 mdelay(10);
1406 }
1407 if (i == 100)
1408 return -ETIMEDOUT;
1409
1410 return 0;
1411 }
1412
cik_set_uvd_clocks(struct amdgpu_device * adev,u32 vclk,u32 dclk)1413 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1414 {
1415 int r = 0;
1416
1417 r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1418 if (r)
1419 return r;
1420
1421 r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1422 return r;
1423 }
1424
cik_set_vce_clocks(struct amdgpu_device * adev,u32 evclk,u32 ecclk)1425 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1426 {
1427 int r, i;
1428 struct atom_clock_dividers dividers;
1429 u32 tmp;
1430
1431 r = amdgpu_atombios_get_clock_dividers(adev,
1432 COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1433 ecclk, false, ÷rs);
1434 if (r)
1435 return r;
1436
1437 for (i = 0; i < 100; i++) {
1438 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1439 break;
1440 mdelay(10);
1441 }
1442 if (i == 100)
1443 return -ETIMEDOUT;
1444
1445 tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1446 tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1447 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1448 tmp |= dividers.post_divider;
1449 WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1450
1451 for (i = 0; i < 100; i++) {
1452 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1453 break;
1454 mdelay(10);
1455 }
1456 if (i == 100)
1457 return -ETIMEDOUT;
1458
1459 return 0;
1460 }
1461
cik_pcie_gen3_enable(struct amdgpu_device * adev)1462 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1463 {
1464 struct pci_dev *root = adev->pdev->bus->self;
1465 u32 speed_cntl, current_data_rate;
1466 int i;
1467 u16 tmp16;
1468
1469 if (pci_is_root_bus(adev->pdev->bus))
1470 return;
1471
1472 if (amdgpu_pcie_gen2 == 0)
1473 return;
1474
1475 if (adev->flags & AMD_IS_APU)
1476 return;
1477
1478 if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1479 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1480 return;
1481
1482 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1483 current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1484 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1485 if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1486 if (current_data_rate == 2) {
1487 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1488 return;
1489 }
1490 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1491 } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1492 if (current_data_rate == 1) {
1493 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1494 return;
1495 }
1496 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1497 }
1498
1499 if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1500 return;
1501
1502 if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1503 /* re-try equalization if gen3 is not already enabled */
1504 if (current_data_rate != 2) {
1505 u16 bridge_cfg, gpu_cfg;
1506 u16 bridge_cfg2, gpu_cfg2;
1507 u32 max_lw, current_lw, tmp;
1508
1509 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1510 &bridge_cfg);
1511 pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1512 &gpu_cfg);
1513
1514 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1515 pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1516
1517 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1518 pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1519 tmp16);
1520
1521 tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1522 max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1523 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1524 current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1525 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1526
1527 if (current_lw < max_lw) {
1528 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1529 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1530 tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1531 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1532 tmp |= (max_lw <<
1533 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1534 tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1535 PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1536 PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1537 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1538 }
1539 }
1540
1541 for (i = 0; i < 10; i++) {
1542 /* check status */
1543 pcie_capability_read_word(adev->pdev,
1544 PCI_EXP_DEVSTA,
1545 &tmp16);
1546 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1547 break;
1548
1549 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1550 &bridge_cfg);
1551 pcie_capability_read_word(adev->pdev,
1552 PCI_EXP_LNKCTL,
1553 &gpu_cfg);
1554
1555 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1556 &bridge_cfg2);
1557 pcie_capability_read_word(adev->pdev,
1558 PCI_EXP_LNKCTL2,
1559 &gpu_cfg2);
1560
1561 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1562 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1563 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1564
1565 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1566 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1567 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1568
1569 msleep(100);
1570
1571 /* linkctl */
1572 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1573 &tmp16);
1574 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1575 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1576 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1577 tmp16);
1578
1579 pcie_capability_read_word(adev->pdev,
1580 PCI_EXP_LNKCTL,
1581 &tmp16);
1582 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1583 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1584 pcie_capability_write_word(adev->pdev,
1585 PCI_EXP_LNKCTL,
1586 tmp16);
1587
1588 /* linkctl2 */
1589 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1590 &tmp16);
1591 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1592 PCI_EXP_LNKCTL2_TX_MARGIN);
1593 tmp16 |= (bridge_cfg2 &
1594 (PCI_EXP_LNKCTL2_ENTER_COMP |
1595 PCI_EXP_LNKCTL2_TX_MARGIN));
1596 pcie_capability_write_word(root,
1597 PCI_EXP_LNKCTL2,
1598 tmp16);
1599
1600 pcie_capability_read_word(adev->pdev,
1601 PCI_EXP_LNKCTL2,
1602 &tmp16);
1603 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1604 PCI_EXP_LNKCTL2_TX_MARGIN);
1605 tmp16 |= (gpu_cfg2 &
1606 (PCI_EXP_LNKCTL2_ENTER_COMP |
1607 PCI_EXP_LNKCTL2_TX_MARGIN));
1608 pcie_capability_write_word(adev->pdev,
1609 PCI_EXP_LNKCTL2,
1610 tmp16);
1611
1612 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1613 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1614 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1615 }
1616 }
1617 }
1618
1619 /* set the link speed */
1620 speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1621 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1622 speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1623 WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1624
1625 pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1626 tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1627
1628 if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1629 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1630 else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1631 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1632 else
1633 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1634 pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1635
1636 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1637 speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1638 WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1639
1640 for (i = 0; i < adev->usec_timeout; i++) {
1641 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1642 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1643 break;
1644 udelay(1);
1645 }
1646 }
1647
cik_program_aspm(struct amdgpu_device * adev)1648 static void cik_program_aspm(struct amdgpu_device *adev)
1649 {
1650 u32 data, orig;
1651 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1652 bool disable_clkreq = false;
1653
1654 if (amdgpu_aspm == 0)
1655 return;
1656
1657 if (pci_is_root_bus(adev->pdev->bus))
1658 return;
1659
1660 /* XXX double check APUs */
1661 if (adev->flags & AMD_IS_APU)
1662 return;
1663
1664 orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1665 data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1666 data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1667 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1668 if (orig != data)
1669 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1670
1671 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1672 data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1673 if (orig != data)
1674 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1675
1676 orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1677 data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1678 if (orig != data)
1679 WREG32_PCIE(ixPCIE_P_CNTL, data);
1680
1681 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1682 data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1683 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1684 data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1685 if (!disable_l0s)
1686 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1687
1688 if (!disable_l1) {
1689 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1690 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1691 if (orig != data)
1692 WREG32_PCIE(ixPCIE_LC_CNTL, data);
1693
1694 if (!disable_plloff_in_l1) {
1695 bool clk_req_support;
1696
1697 orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1698 data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1699 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1700 data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1701 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1702 if (orig != data)
1703 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1704
1705 orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1706 data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1707 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1708 data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1709 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1710 if (orig != data)
1711 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1712
1713 orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1714 data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1715 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1716 data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1717 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1718 if (orig != data)
1719 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1720
1721 orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1722 data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1723 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1724 data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1725 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1726 if (orig != data)
1727 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1728
1729 orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1730 data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1731 data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1732 if (orig != data)
1733 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1734
1735 if (!disable_clkreq) {
1736 struct pci_dev *root = adev->pdev->bus->self;
1737 u32 lnkcap;
1738
1739 clk_req_support = false;
1740 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1741 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1742 clk_req_support = true;
1743 } else {
1744 clk_req_support = false;
1745 }
1746
1747 if (clk_req_support) {
1748 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1749 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1750 PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1751 if (orig != data)
1752 WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1753
1754 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1755 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1756 THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1757 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1758 (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1759 if (orig != data)
1760 WREG32_SMC(ixTHM_CLK_CNTL, data);
1761
1762 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1763 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1764 MISC_CLK_CTRL__ZCLK_SEL_MASK);
1765 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1766 (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1767 if (orig != data)
1768 WREG32_SMC(ixMISC_CLK_CTRL, data);
1769
1770 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1771 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1772 if (orig != data)
1773 WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1774
1775 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1776 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1777 if (orig != data)
1778 WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1779
1780 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1781 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1782 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1783 if (orig != data)
1784 WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1785 }
1786 }
1787 } else {
1788 if (orig != data)
1789 WREG32_PCIE(ixPCIE_LC_CNTL, data);
1790 }
1791
1792 orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1793 data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1794 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1795 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1796 if (orig != data)
1797 WREG32_PCIE(ixPCIE_CNTL2, data);
1798
1799 if (!disable_l0s) {
1800 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1801 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1802 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1803 data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1804 if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1805 (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1806 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1807 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1808 if (orig != data)
1809 WREG32_PCIE(ixPCIE_LC_CNTL, data);
1810 }
1811 }
1812 }
1813 }
1814
cik_get_rev_id(struct amdgpu_device * adev)1815 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1816 {
1817 return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1818 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1819 }
1820
cik_detect_hw_virtualization(struct amdgpu_device * adev)1821 static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
1822 {
1823 if (is_virtual_machine()) /* passthrough mode */
1824 adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1825 }
1826
cik_flush_hdp(struct amdgpu_device * adev,struct amdgpu_ring * ring)1827 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1828 {
1829 if (!ring || !ring->funcs->emit_wreg) {
1830 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1831 RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1832 } else {
1833 amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1834 }
1835 }
1836
cik_invalidate_hdp(struct amdgpu_device * adev,struct amdgpu_ring * ring)1837 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1838 struct amdgpu_ring *ring)
1839 {
1840 if (!ring || !ring->funcs->emit_wreg) {
1841 WREG32(mmHDP_DEBUG0, 1);
1842 RREG32(mmHDP_DEBUG0);
1843 } else {
1844 amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1845 }
1846 }
1847
cik_need_full_reset(struct amdgpu_device * adev)1848 static bool cik_need_full_reset(struct amdgpu_device *adev)
1849 {
1850 /* change this when we support soft reset */
1851 return true;
1852 }
1853
cik_get_pcie_usage(struct amdgpu_device * adev,uint64_t * count0,uint64_t * count1)1854 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1855 uint64_t *count1)
1856 {
1857 uint32_t perfctr = 0;
1858 uint64_t cnt0_of, cnt1_of;
1859 int tmp;
1860
1861 /* This reports 0 on APUs, so return to avoid writing/reading registers
1862 * that may or may not be different from their GPU counterparts
1863 */
1864 if (adev->flags & AMD_IS_APU)
1865 return;
1866
1867 /* Set the 2 events that we wish to watch, defined above */
1868 /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1869 perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1870 perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1871
1872 /* Write to enable desired perf counters */
1873 WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1874 /* Zero out and enable the perf counters
1875 * Write 0x5:
1876 * Bit 0 = Start all counters(1)
1877 * Bit 2 = Global counter reset enable(1)
1878 */
1879 WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1880
1881 msleep(1000);
1882
1883 /* Load the shadow and disable the perf counters
1884 * Write 0x2:
1885 * Bit 0 = Stop counters(0)
1886 * Bit 1 = Load the shadow counters(1)
1887 */
1888 WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1889
1890 /* Read register values to get any >32bit overflow */
1891 tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1892 cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1893 cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1894
1895 /* Get the values and add the overflow */
1896 *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1897 *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1898 }
1899
cik_need_reset_on_init(struct amdgpu_device * adev)1900 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1901 {
1902 u32 clock_cntl, pc;
1903
1904 if (adev->flags & AMD_IS_APU)
1905 return false;
1906
1907 /* check if the SMC is already running */
1908 clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1909 pc = RREG32_SMC(ixSMC_PC_C);
1910 if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1911 (0x20100 <= pc))
1912 return true;
1913
1914 return false;
1915 }
1916
cik_get_pcie_replay_count(struct amdgpu_device * adev)1917 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1918 {
1919 uint64_t nak_r, nak_g;
1920
1921 /* Get the number of NAKs received and generated */
1922 nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1923 nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1924
1925 /* Add the total number of NAKs, i.e the number of replays */
1926 return (nak_r + nak_g);
1927 }
1928
1929 static const struct amdgpu_asic_funcs cik_asic_funcs =
1930 {
1931 .read_disabled_bios = &cik_read_disabled_bios,
1932 .read_bios_from_rom = &cik_read_bios_from_rom,
1933 .read_register = &cik_read_register,
1934 .reset = &cik_asic_reset,
1935 .reset_method = &cik_asic_reset_method,
1936 .set_vga_state = &cik_vga_set_state,
1937 .get_xclk = &cik_get_xclk,
1938 .set_uvd_clocks = &cik_set_uvd_clocks,
1939 .set_vce_clocks = &cik_set_vce_clocks,
1940 .get_config_memsize = &cik_get_config_memsize,
1941 .flush_hdp = &cik_flush_hdp,
1942 .invalidate_hdp = &cik_invalidate_hdp,
1943 .need_full_reset = &cik_need_full_reset,
1944 .init_doorbell_index = &legacy_doorbell_index_init,
1945 .get_pcie_usage = &cik_get_pcie_usage,
1946 .need_reset_on_init = &cik_need_reset_on_init,
1947 .get_pcie_replay_count = &cik_get_pcie_replay_count,
1948 .supports_baco = &cik_asic_supports_baco,
1949 };
1950
cik_common_early_init(void * handle)1951 static int cik_common_early_init(void *handle)
1952 {
1953 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1954
1955 adev->smc_rreg = &cik_smc_rreg;
1956 adev->smc_wreg = &cik_smc_wreg;
1957 adev->pcie_rreg = &cik_pcie_rreg;
1958 adev->pcie_wreg = &cik_pcie_wreg;
1959 adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1960 adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1961 adev->didt_rreg = &cik_didt_rreg;
1962 adev->didt_wreg = &cik_didt_wreg;
1963
1964 adev->asic_funcs = &cik_asic_funcs;
1965
1966 adev->rev_id = cik_get_rev_id(adev);
1967 adev->external_rev_id = 0xFF;
1968 switch (adev->asic_type) {
1969 case CHIP_BONAIRE:
1970 adev->cg_flags =
1971 AMD_CG_SUPPORT_GFX_MGCG |
1972 AMD_CG_SUPPORT_GFX_MGLS |
1973 /*AMD_CG_SUPPORT_GFX_CGCG |*/
1974 AMD_CG_SUPPORT_GFX_CGLS |
1975 AMD_CG_SUPPORT_GFX_CGTS |
1976 AMD_CG_SUPPORT_GFX_CGTS_LS |
1977 AMD_CG_SUPPORT_GFX_CP_LS |
1978 AMD_CG_SUPPORT_MC_LS |
1979 AMD_CG_SUPPORT_MC_MGCG |
1980 AMD_CG_SUPPORT_SDMA_MGCG |
1981 AMD_CG_SUPPORT_SDMA_LS |
1982 AMD_CG_SUPPORT_BIF_LS |
1983 AMD_CG_SUPPORT_VCE_MGCG |
1984 AMD_CG_SUPPORT_UVD_MGCG |
1985 AMD_CG_SUPPORT_HDP_LS |
1986 AMD_CG_SUPPORT_HDP_MGCG;
1987 adev->pg_flags = 0;
1988 adev->external_rev_id = adev->rev_id + 0x14;
1989 break;
1990 case CHIP_HAWAII:
1991 adev->cg_flags =
1992 AMD_CG_SUPPORT_GFX_MGCG |
1993 AMD_CG_SUPPORT_GFX_MGLS |
1994 /*AMD_CG_SUPPORT_GFX_CGCG |*/
1995 AMD_CG_SUPPORT_GFX_CGLS |
1996 AMD_CG_SUPPORT_GFX_CGTS |
1997 AMD_CG_SUPPORT_GFX_CP_LS |
1998 AMD_CG_SUPPORT_MC_LS |
1999 AMD_CG_SUPPORT_MC_MGCG |
2000 AMD_CG_SUPPORT_SDMA_MGCG |
2001 AMD_CG_SUPPORT_SDMA_LS |
2002 AMD_CG_SUPPORT_BIF_LS |
2003 AMD_CG_SUPPORT_VCE_MGCG |
2004 AMD_CG_SUPPORT_UVD_MGCG |
2005 AMD_CG_SUPPORT_HDP_LS |
2006 AMD_CG_SUPPORT_HDP_MGCG;
2007 adev->pg_flags = 0;
2008 adev->external_rev_id = 0x28;
2009 break;
2010 case CHIP_KAVERI:
2011 adev->cg_flags =
2012 AMD_CG_SUPPORT_GFX_MGCG |
2013 AMD_CG_SUPPORT_GFX_MGLS |
2014 /*AMD_CG_SUPPORT_GFX_CGCG |*/
2015 AMD_CG_SUPPORT_GFX_CGLS |
2016 AMD_CG_SUPPORT_GFX_CGTS |
2017 AMD_CG_SUPPORT_GFX_CGTS_LS |
2018 AMD_CG_SUPPORT_GFX_CP_LS |
2019 AMD_CG_SUPPORT_SDMA_MGCG |
2020 AMD_CG_SUPPORT_SDMA_LS |
2021 AMD_CG_SUPPORT_BIF_LS |
2022 AMD_CG_SUPPORT_VCE_MGCG |
2023 AMD_CG_SUPPORT_UVD_MGCG |
2024 AMD_CG_SUPPORT_HDP_LS |
2025 AMD_CG_SUPPORT_HDP_MGCG;
2026 adev->pg_flags =
2027 /*AMD_PG_SUPPORT_GFX_PG |
2028 AMD_PG_SUPPORT_GFX_SMG |
2029 AMD_PG_SUPPORT_GFX_DMG |*/
2030 AMD_PG_SUPPORT_UVD |
2031 AMD_PG_SUPPORT_VCE |
2032 /* AMD_PG_SUPPORT_CP |
2033 AMD_PG_SUPPORT_GDS |
2034 AMD_PG_SUPPORT_RLC_SMU_HS |
2035 AMD_PG_SUPPORT_ACP |
2036 AMD_PG_SUPPORT_SAMU |*/
2037 0;
2038 if (adev->pdev->device == 0x1312 ||
2039 adev->pdev->device == 0x1316 ||
2040 adev->pdev->device == 0x1317)
2041 adev->external_rev_id = 0x41;
2042 else
2043 adev->external_rev_id = 0x1;
2044 break;
2045 case CHIP_KABINI:
2046 case CHIP_MULLINS:
2047 adev->cg_flags =
2048 AMD_CG_SUPPORT_GFX_MGCG |
2049 AMD_CG_SUPPORT_GFX_MGLS |
2050 /*AMD_CG_SUPPORT_GFX_CGCG |*/
2051 AMD_CG_SUPPORT_GFX_CGLS |
2052 AMD_CG_SUPPORT_GFX_CGTS |
2053 AMD_CG_SUPPORT_GFX_CGTS_LS |
2054 AMD_CG_SUPPORT_GFX_CP_LS |
2055 AMD_CG_SUPPORT_SDMA_MGCG |
2056 AMD_CG_SUPPORT_SDMA_LS |
2057 AMD_CG_SUPPORT_BIF_LS |
2058 AMD_CG_SUPPORT_VCE_MGCG |
2059 AMD_CG_SUPPORT_UVD_MGCG |
2060 AMD_CG_SUPPORT_HDP_LS |
2061 AMD_CG_SUPPORT_HDP_MGCG;
2062 adev->pg_flags =
2063 /*AMD_PG_SUPPORT_GFX_PG |
2064 AMD_PG_SUPPORT_GFX_SMG | */
2065 AMD_PG_SUPPORT_UVD |
2066 /*AMD_PG_SUPPORT_VCE |
2067 AMD_PG_SUPPORT_CP |
2068 AMD_PG_SUPPORT_GDS |
2069 AMD_PG_SUPPORT_RLC_SMU_HS |
2070 AMD_PG_SUPPORT_SAMU |*/
2071 0;
2072 if (adev->asic_type == CHIP_KABINI) {
2073 if (adev->rev_id == 0)
2074 adev->external_rev_id = 0x81;
2075 else if (adev->rev_id == 1)
2076 adev->external_rev_id = 0x82;
2077 else if (adev->rev_id == 2)
2078 adev->external_rev_id = 0x85;
2079 } else
2080 adev->external_rev_id = adev->rev_id + 0xa1;
2081 break;
2082 default:
2083 /* FIXME: not supported yet */
2084 return -EINVAL;
2085 }
2086
2087 return 0;
2088 }
2089
cik_common_sw_init(void * handle)2090 static int cik_common_sw_init(void *handle)
2091 {
2092 return 0;
2093 }
2094
cik_common_sw_fini(void * handle)2095 static int cik_common_sw_fini(void *handle)
2096 {
2097 return 0;
2098 }
2099
cik_common_hw_init(void * handle)2100 static int cik_common_hw_init(void *handle)
2101 {
2102 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2103
2104 /* move the golden regs per IP block */
2105 cik_init_golden_registers(adev);
2106 /* enable pcie gen2/3 link */
2107 cik_pcie_gen3_enable(adev);
2108 /* enable aspm */
2109 cik_program_aspm(adev);
2110
2111 return 0;
2112 }
2113
cik_common_hw_fini(void * handle)2114 static int cik_common_hw_fini(void *handle)
2115 {
2116 return 0;
2117 }
2118
cik_common_suspend(void * handle)2119 static int cik_common_suspend(void *handle)
2120 {
2121 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2122
2123 return cik_common_hw_fini(adev);
2124 }
2125
cik_common_resume(void * handle)2126 static int cik_common_resume(void *handle)
2127 {
2128 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2129
2130 return cik_common_hw_init(adev);
2131 }
2132
cik_common_is_idle(void * handle)2133 static bool cik_common_is_idle(void *handle)
2134 {
2135 return true;
2136 }
2137
cik_common_wait_for_idle(void * handle)2138 static int cik_common_wait_for_idle(void *handle)
2139 {
2140 return 0;
2141 }
2142
cik_common_soft_reset(void * handle)2143 static int cik_common_soft_reset(void *handle)
2144 {
2145 /* XXX hard reset?? */
2146 return 0;
2147 }
2148
cik_common_set_clockgating_state(void * handle,enum amd_clockgating_state state)2149 static int cik_common_set_clockgating_state(void *handle,
2150 enum amd_clockgating_state state)
2151 {
2152 return 0;
2153 }
2154
cik_common_set_powergating_state(void * handle,enum amd_powergating_state state)2155 static int cik_common_set_powergating_state(void *handle,
2156 enum amd_powergating_state state)
2157 {
2158 return 0;
2159 }
2160
2161 static const struct amd_ip_funcs cik_common_ip_funcs = {
2162 .name = "cik_common",
2163 .early_init = cik_common_early_init,
2164 .late_init = NULL,
2165 .sw_init = cik_common_sw_init,
2166 .sw_fini = cik_common_sw_fini,
2167 .hw_init = cik_common_hw_init,
2168 .hw_fini = cik_common_hw_fini,
2169 .suspend = cik_common_suspend,
2170 .resume = cik_common_resume,
2171 .is_idle = cik_common_is_idle,
2172 .wait_for_idle = cik_common_wait_for_idle,
2173 .soft_reset = cik_common_soft_reset,
2174 .set_clockgating_state = cik_common_set_clockgating_state,
2175 .set_powergating_state = cik_common_set_powergating_state,
2176 };
2177
2178 static const struct amdgpu_ip_block_version cik_common_ip_block =
2179 {
2180 .type = AMD_IP_BLOCK_TYPE_COMMON,
2181 .major = 1,
2182 .minor = 0,
2183 .rev = 0,
2184 .funcs = &cik_common_ip_funcs,
2185 };
2186
cik_set_ip_blocks(struct amdgpu_device * adev)2187 int cik_set_ip_blocks(struct amdgpu_device *adev)
2188 {
2189 cik_detect_hw_virtualization(adev);
2190
2191 switch (adev->asic_type) {
2192 case CHIP_BONAIRE:
2193 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2194 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2195 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2196 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2197 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2198 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2199 if (adev->enable_virtual_display)
2200 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2201 #if defined(CONFIG_DRM_AMD_DC)
2202 else if (amdgpu_device_has_dc_support(adev))
2203 amdgpu_device_ip_block_add(adev, &dm_ip_block);
2204 #endif
2205 else
2206 amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2207 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2208 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2209 break;
2210 case CHIP_HAWAII:
2211 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2212 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2213 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2214 amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2215 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2216 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2217 if (adev->enable_virtual_display)
2218 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2219 #if defined(CONFIG_DRM_AMD_DC)
2220 else if (amdgpu_device_has_dc_support(adev))
2221 amdgpu_device_ip_block_add(adev, &dm_ip_block);
2222 #endif
2223 else
2224 amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2225 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2226 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2227 break;
2228 case CHIP_KAVERI:
2229 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2230 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2231 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2232 amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2233 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2234 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2235 if (adev->enable_virtual_display)
2236 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2237 #if defined(CONFIG_DRM_AMD_DC)
2238 else if (amdgpu_device_has_dc_support(adev))
2239 amdgpu_device_ip_block_add(adev, &dm_ip_block);
2240 #endif
2241 else
2242 amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2243
2244 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2245 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2246 break;
2247 case CHIP_KABINI:
2248 case CHIP_MULLINS:
2249 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2250 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2251 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2252 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2253 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2254 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2255 if (adev->enable_virtual_display)
2256 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2257 #if defined(CONFIG_DRM_AMD_DC)
2258 else if (amdgpu_device_has_dc_support(adev))
2259 amdgpu_device_ip_block_add(adev, &dm_ip_block);
2260 #endif
2261 else
2262 amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2263 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2264 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2265 break;
2266 default:
2267 /* FIXME: not supported yet */
2268 return -EINVAL;
2269 }
2270 return 0;
2271 }
2272