xref: /netbsd-src/sys/external/bsd/drm2/dist/drm/amd/amdgpu/amdgpu_cik.c (revision 53b02e147d4ed531c0d2a5ca9b3e8026ba3e99b5) 
1 /*	$NetBSD: amdgpu_cik.c,v 1.5 2021/12/19 10:59:01 riastradh Exp $	*/
2 
3 /*
4  * Copyright 2012 Advanced Micro Devices, Inc.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Alex Deucher
25  */
26 #include <sys/cdefs.h>
27 __KERNEL_RCSID(0, "$NetBSD: amdgpu_cik.c,v 1.5 2021/12/19 10:59:01 riastradh Exp $");
28 
29 #include <linux/firmware.h>
30 #include <linux/slab.h>
31 #include <linux/module.h>
32 #include <linux/pci.h>
33 
34 #include "amdgpu.h"
35 #include "amdgpu_atombios.h"
36 #include "amdgpu_ih.h"
37 #include "amdgpu_uvd.h"
38 #include "amdgpu_vce.h"
39 #include "cikd.h"
40 #include "atom.h"
41 #include "amd_pcie.h"
42 
43 #include "cik.h"
44 #include "gmc_v7_0.h"
45 #include "cik_ih.h"
46 #include "dce_v8_0.h"
47 #include "gfx_v7_0.h"
48 #include "cik_sdma.h"
49 #include "uvd_v4_2.h"
50 #include "vce_v2_0.h"
51 #include "cik_dpm.h"
52 
53 #include "uvd/uvd_4_2_d.h"
54 
55 #include "smu/smu_7_0_1_d.h"
56 #include "smu/smu_7_0_1_sh_mask.h"
57 
58 #include "dce/dce_8_0_d.h"
59 #include "dce/dce_8_0_sh_mask.h"
60 
61 #include "bif/bif_4_1_d.h"
62 #include "bif/bif_4_1_sh_mask.h"
63 
64 #include "gca/gfx_7_2_d.h"
65 #include "gca/gfx_7_2_enum.h"
66 #include "gca/gfx_7_2_sh_mask.h"
67 
68 #include "gmc/gmc_7_1_d.h"
69 #include "gmc/gmc_7_1_sh_mask.h"
70 
71 #include "oss/oss_2_0_d.h"
72 #include "oss/oss_2_0_sh_mask.h"
73 
74 #include "amdgpu_dm.h"
75 #include "amdgpu_amdkfd.h"
76 #include "dce_virtual.h"
77 
78 #include <linux/nbsd-namespace.h>
79 
80 /*
81  * Indirect registers accessor
82  */
83 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
84 {
85 	unsigned long flags;
86 	u32 r;
87 
88 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
89 	WREG32(mmPCIE_INDEX, reg);
90 	(void)RREG32(mmPCIE_INDEX);
91 	r = RREG32(mmPCIE_DATA);
92 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
93 	return r;
94 }
95 
96 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
97 {
98 	unsigned long flags;
99 
100 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
101 	WREG32(mmPCIE_INDEX, reg);
102 	(void)RREG32(mmPCIE_INDEX);
103 	WREG32(mmPCIE_DATA, v);
104 	(void)RREG32(mmPCIE_DATA);
105 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
106 }
107 
108 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
109 {
110 	unsigned long flags;
111 	u32 r;
112 
113 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
114 	WREG32(mmSMC_IND_INDEX_0, (reg));
115 	r = RREG32(mmSMC_IND_DATA_0);
116 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
117 	return r;
118 }
119 
120 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
121 {
122 	unsigned long flags;
123 
124 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
125 	WREG32(mmSMC_IND_INDEX_0, (reg));
126 	WREG32(mmSMC_IND_DATA_0, (v));
127 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
128 }
129 
130 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
131 {
132 	unsigned long flags;
133 	u32 r;
134 
135 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
136 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
137 	r = RREG32(mmUVD_CTX_DATA);
138 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
139 	return r;
140 }
141 
142 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
143 {
144 	unsigned long flags;
145 
146 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
147 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
148 	WREG32(mmUVD_CTX_DATA, (v));
149 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
150 }
151 
152 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
153 {
154 	unsigned long flags;
155 	u32 r;
156 
157 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
158 	WREG32(mmDIDT_IND_INDEX, (reg));
159 	r = RREG32(mmDIDT_IND_DATA);
160 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
161 	return r;
162 }
163 
164 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
165 {
166 	unsigned long flags;
167 
168 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
169 	WREG32(mmDIDT_IND_INDEX, (reg));
170 	WREG32(mmDIDT_IND_DATA, (v));
171 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
172 }
173 
174 static const u32 bonaire_golden_spm_registers[] =
175 {
176 	0xc200, 0xe0ffffff, 0xe0000000
177 };
178 
179 static const u32 bonaire_golden_common_registers[] =
180 {
181 	0x31dc, 0xffffffff, 0x00000800,
182 	0x31dd, 0xffffffff, 0x00000800,
183 	0x31e6, 0xffffffff, 0x00007fbf,
184 	0x31e7, 0xffffffff, 0x00007faf
185 };
186 
187 static const u32 bonaire_golden_registers[] =
188 {
189 	0xcd5, 0x00000333, 0x00000333,
190 	0xcd4, 0x000c0fc0, 0x00040200,
191 	0x2684, 0x00010000, 0x00058208,
192 	0xf000, 0xffff1fff, 0x00140000,
193 	0xf080, 0xfdfc0fff, 0x00000100,
194 	0xf08d, 0x40000000, 0x40000200,
195 	0x260c, 0xffffffff, 0x00000000,
196 	0x260d, 0xf00fffff, 0x00000400,
197 	0x260e, 0x0002021c, 0x00020200,
198 	0x31e, 0x00000080, 0x00000000,
199 	0x16ec, 0x000000f0, 0x00000070,
200 	0x16f0, 0xf0311fff, 0x80300000,
201 	0x263e, 0x73773777, 0x12010001,
202 	0xd43, 0x00810000, 0x408af000,
203 	0x1c0c, 0x31000111, 0x00000011,
204 	0xbd2, 0x73773777, 0x12010001,
205 	0x883, 0x00007fb6, 0x0021a1b1,
206 	0x884, 0x00007fb6, 0x002021b1,
207 	0x860, 0x00007fb6, 0x00002191,
208 	0x886, 0x00007fb6, 0x002121b1,
209 	0x887, 0x00007fb6, 0x002021b1,
210 	0x877, 0x00007fb6, 0x00002191,
211 	0x878, 0x00007fb6, 0x00002191,
212 	0xd8a, 0x0000003f, 0x0000000a,
213 	0xd8b, 0x0000003f, 0x0000000a,
214 	0xab9, 0x00073ffe, 0x000022a2,
215 	0x903, 0x000007ff, 0x00000000,
216 	0x2285, 0xf000003f, 0x00000007,
217 	0x22fc, 0x00002001, 0x00000001,
218 	0x22c9, 0xffffffff, 0x00ffffff,
219 	0xc281, 0x0000ff0f, 0x00000000,
220 	0xa293, 0x07ffffff, 0x06000000,
221 	0x136, 0x00000fff, 0x00000100,
222 	0xf9e, 0x00000001, 0x00000002,
223 	0x2440, 0x03000000, 0x0362c688,
224 	0x2300, 0x000000ff, 0x00000001,
225 	0x390, 0x00001fff, 0x00001fff,
226 	0x2418, 0x0000007f, 0x00000020,
227 	0x2542, 0x00010000, 0x00010000,
228 	0x2b05, 0x000003ff, 0x000000f3,
229 	0x2b03, 0xffffffff, 0x00001032
230 };
231 
232 static const u32 bonaire_mgcg_cgcg_init[] =
233 {
234 	0x3108, 0xffffffff, 0xfffffffc,
235 	0xc200, 0xffffffff, 0xe0000000,
236 	0xf0a8, 0xffffffff, 0x00000100,
237 	0xf082, 0xffffffff, 0x00000100,
238 	0xf0b0, 0xffffffff, 0xc0000100,
239 	0xf0b2, 0xffffffff, 0xc0000100,
240 	0xf0b1, 0xffffffff, 0xc0000100,
241 	0x1579, 0xffffffff, 0x00600100,
242 	0xf0a0, 0xffffffff, 0x00000100,
243 	0xf085, 0xffffffff, 0x06000100,
244 	0xf088, 0xffffffff, 0x00000100,
245 	0xf086, 0xffffffff, 0x06000100,
246 	0xf081, 0xffffffff, 0x00000100,
247 	0xf0b8, 0xffffffff, 0x00000100,
248 	0xf089, 0xffffffff, 0x00000100,
249 	0xf080, 0xffffffff, 0x00000100,
250 	0xf08c, 0xffffffff, 0x00000100,
251 	0xf08d, 0xffffffff, 0x00000100,
252 	0xf094, 0xffffffff, 0x00000100,
253 	0xf095, 0xffffffff, 0x00000100,
254 	0xf096, 0xffffffff, 0x00000100,
255 	0xf097, 0xffffffff, 0x00000100,
256 	0xf098, 0xffffffff, 0x00000100,
257 	0xf09f, 0xffffffff, 0x00000100,
258 	0xf09e, 0xffffffff, 0x00000100,
259 	0xf084, 0xffffffff, 0x06000100,
260 	0xf0a4, 0xffffffff, 0x00000100,
261 	0xf09d, 0xffffffff, 0x00000100,
262 	0xf0ad, 0xffffffff, 0x00000100,
263 	0xf0ac, 0xffffffff, 0x00000100,
264 	0xf09c, 0xffffffff, 0x00000100,
265 	0xc200, 0xffffffff, 0xe0000000,
266 	0xf008, 0xffffffff, 0x00010000,
267 	0xf009, 0xffffffff, 0x00030002,
268 	0xf00a, 0xffffffff, 0x00040007,
269 	0xf00b, 0xffffffff, 0x00060005,
270 	0xf00c, 0xffffffff, 0x00090008,
271 	0xf00d, 0xffffffff, 0x00010000,
272 	0xf00e, 0xffffffff, 0x00030002,
273 	0xf00f, 0xffffffff, 0x00040007,
274 	0xf010, 0xffffffff, 0x00060005,
275 	0xf011, 0xffffffff, 0x00090008,
276 	0xf012, 0xffffffff, 0x00010000,
277 	0xf013, 0xffffffff, 0x00030002,
278 	0xf014, 0xffffffff, 0x00040007,
279 	0xf015, 0xffffffff, 0x00060005,
280 	0xf016, 0xffffffff, 0x00090008,
281 	0xf017, 0xffffffff, 0x00010000,
282 	0xf018, 0xffffffff, 0x00030002,
283 	0xf019, 0xffffffff, 0x00040007,
284 	0xf01a, 0xffffffff, 0x00060005,
285 	0xf01b, 0xffffffff, 0x00090008,
286 	0xf01c, 0xffffffff, 0x00010000,
287 	0xf01d, 0xffffffff, 0x00030002,
288 	0xf01e, 0xffffffff, 0x00040007,
289 	0xf01f, 0xffffffff, 0x00060005,
290 	0xf020, 0xffffffff, 0x00090008,
291 	0xf021, 0xffffffff, 0x00010000,
292 	0xf022, 0xffffffff, 0x00030002,
293 	0xf023, 0xffffffff, 0x00040007,
294 	0xf024, 0xffffffff, 0x00060005,
295 	0xf025, 0xffffffff, 0x00090008,
296 	0xf026, 0xffffffff, 0x00010000,
297 	0xf027, 0xffffffff, 0x00030002,
298 	0xf028, 0xffffffff, 0x00040007,
299 	0xf029, 0xffffffff, 0x00060005,
300 	0xf02a, 0xffffffff, 0x00090008,
301 	0xf000, 0xffffffff, 0x96e00200,
302 	0x21c2, 0xffffffff, 0x00900100,
303 	0x3109, 0xffffffff, 0x0020003f,
304 	0xe, 0xffffffff, 0x0140001c,
305 	0xf, 0x000f0000, 0x000f0000,
306 	0x88, 0xffffffff, 0xc060000c,
307 	0x89, 0xc0000fff, 0x00000100,
308 	0x3e4, 0xffffffff, 0x00000100,
309 	0x3e6, 0x00000101, 0x00000000,
310 	0x82a, 0xffffffff, 0x00000104,
311 	0x1579, 0xff000fff, 0x00000100,
312 	0xc33, 0xc0000fff, 0x00000104,
313 	0x3079, 0x00000001, 0x00000001,
314 	0x3403, 0xff000ff0, 0x00000100,
315 	0x3603, 0xff000ff0, 0x00000100
316 };
317 
318 static const u32 spectre_golden_spm_registers[] =
319 {
320 	0xc200, 0xe0ffffff, 0xe0000000
321 };
322 
323 static const u32 spectre_golden_common_registers[] =
324 {
325 	0x31dc, 0xffffffff, 0x00000800,
326 	0x31dd, 0xffffffff, 0x00000800,
327 	0x31e6, 0xffffffff, 0x00007fbf,
328 	0x31e7, 0xffffffff, 0x00007faf
329 };
330 
331 static const u32 spectre_golden_registers[] =
332 {
333 	0xf000, 0xffff1fff, 0x96940200,
334 	0xf003, 0xffff0001, 0xff000000,
335 	0xf080, 0xfffc0fff, 0x00000100,
336 	0x1bb6, 0x00010101, 0x00010000,
337 	0x260d, 0xf00fffff, 0x00000400,
338 	0x260e, 0xfffffffc, 0x00020200,
339 	0x16ec, 0x000000f0, 0x00000070,
340 	0x16f0, 0xf0311fff, 0x80300000,
341 	0x263e, 0x73773777, 0x12010001,
342 	0x26df, 0x00ff0000, 0x00fc0000,
343 	0xbd2, 0x73773777, 0x12010001,
344 	0x2285, 0xf000003f, 0x00000007,
345 	0x22c9, 0xffffffff, 0x00ffffff,
346 	0xa0d4, 0x3f3f3fff, 0x00000082,
347 	0xa0d5, 0x0000003f, 0x00000000,
348 	0xf9e, 0x00000001, 0x00000002,
349 	0x244f, 0xffff03df, 0x00000004,
350 	0x31da, 0x00000008, 0x00000008,
351 	0x2300, 0x000008ff, 0x00000800,
352 	0x2542, 0x00010000, 0x00010000,
353 	0x2b03, 0xffffffff, 0x54763210,
354 	0x853e, 0x01ff01ff, 0x00000002,
355 	0x8526, 0x007ff800, 0x00200000,
356 	0x8057, 0xffffffff, 0x00000f40,
357 	0xc24d, 0xffffffff, 0x00000001
358 };
359 
360 static const u32 spectre_mgcg_cgcg_init[] =
361 {
362 	0x3108, 0xffffffff, 0xfffffffc,
363 	0xc200, 0xffffffff, 0xe0000000,
364 	0xf0a8, 0xffffffff, 0x00000100,
365 	0xf082, 0xffffffff, 0x00000100,
366 	0xf0b0, 0xffffffff, 0x00000100,
367 	0xf0b2, 0xffffffff, 0x00000100,
368 	0xf0b1, 0xffffffff, 0x00000100,
369 	0x1579, 0xffffffff, 0x00600100,
370 	0xf0a0, 0xffffffff, 0x00000100,
371 	0xf085, 0xffffffff, 0x06000100,
372 	0xf088, 0xffffffff, 0x00000100,
373 	0xf086, 0xffffffff, 0x06000100,
374 	0xf081, 0xffffffff, 0x00000100,
375 	0xf0b8, 0xffffffff, 0x00000100,
376 	0xf089, 0xffffffff, 0x00000100,
377 	0xf080, 0xffffffff, 0x00000100,
378 	0xf08c, 0xffffffff, 0x00000100,
379 	0xf08d, 0xffffffff, 0x00000100,
380 	0xf094, 0xffffffff, 0x00000100,
381 	0xf095, 0xffffffff, 0x00000100,
382 	0xf096, 0xffffffff, 0x00000100,
383 	0xf097, 0xffffffff, 0x00000100,
384 	0xf098, 0xffffffff, 0x00000100,
385 	0xf09f, 0xffffffff, 0x00000100,
386 	0xf09e, 0xffffffff, 0x00000100,
387 	0xf084, 0xffffffff, 0x06000100,
388 	0xf0a4, 0xffffffff, 0x00000100,
389 	0xf09d, 0xffffffff, 0x00000100,
390 	0xf0ad, 0xffffffff, 0x00000100,
391 	0xf0ac, 0xffffffff, 0x00000100,
392 	0xf09c, 0xffffffff, 0x00000100,
393 	0xc200, 0xffffffff, 0xe0000000,
394 	0xf008, 0xffffffff, 0x00010000,
395 	0xf009, 0xffffffff, 0x00030002,
396 	0xf00a, 0xffffffff, 0x00040007,
397 	0xf00b, 0xffffffff, 0x00060005,
398 	0xf00c, 0xffffffff, 0x00090008,
399 	0xf00d, 0xffffffff, 0x00010000,
400 	0xf00e, 0xffffffff, 0x00030002,
401 	0xf00f, 0xffffffff, 0x00040007,
402 	0xf010, 0xffffffff, 0x00060005,
403 	0xf011, 0xffffffff, 0x00090008,
404 	0xf012, 0xffffffff, 0x00010000,
405 	0xf013, 0xffffffff, 0x00030002,
406 	0xf014, 0xffffffff, 0x00040007,
407 	0xf015, 0xffffffff, 0x00060005,
408 	0xf016, 0xffffffff, 0x00090008,
409 	0xf017, 0xffffffff, 0x00010000,
410 	0xf018, 0xffffffff, 0x00030002,
411 	0xf019, 0xffffffff, 0x00040007,
412 	0xf01a, 0xffffffff, 0x00060005,
413 	0xf01b, 0xffffffff, 0x00090008,
414 	0xf01c, 0xffffffff, 0x00010000,
415 	0xf01d, 0xffffffff, 0x00030002,
416 	0xf01e, 0xffffffff, 0x00040007,
417 	0xf01f, 0xffffffff, 0x00060005,
418 	0xf020, 0xffffffff, 0x00090008,
419 	0xf021, 0xffffffff, 0x00010000,
420 	0xf022, 0xffffffff, 0x00030002,
421 	0xf023, 0xffffffff, 0x00040007,
422 	0xf024, 0xffffffff, 0x00060005,
423 	0xf025, 0xffffffff, 0x00090008,
424 	0xf026, 0xffffffff, 0x00010000,
425 	0xf027, 0xffffffff, 0x00030002,
426 	0xf028, 0xffffffff, 0x00040007,
427 	0xf029, 0xffffffff, 0x00060005,
428 	0xf02a, 0xffffffff, 0x00090008,
429 	0xf02b, 0xffffffff, 0x00010000,
430 	0xf02c, 0xffffffff, 0x00030002,
431 	0xf02d, 0xffffffff, 0x00040007,
432 	0xf02e, 0xffffffff, 0x00060005,
433 	0xf02f, 0xffffffff, 0x00090008,
434 	0xf000, 0xffffffff, 0x96e00200,
435 	0x21c2, 0xffffffff, 0x00900100,
436 	0x3109, 0xffffffff, 0x0020003f,
437 	0xe, 0xffffffff, 0x0140001c,
438 	0xf, 0x000f0000, 0x000f0000,
439 	0x88, 0xffffffff, 0xc060000c,
440 	0x89, 0xc0000fff, 0x00000100,
441 	0x3e4, 0xffffffff, 0x00000100,
442 	0x3e6, 0x00000101, 0x00000000,
443 	0x82a, 0xffffffff, 0x00000104,
444 	0x1579, 0xff000fff, 0x00000100,
445 	0xc33, 0xc0000fff, 0x00000104,
446 	0x3079, 0x00000001, 0x00000001,
447 	0x3403, 0xff000ff0, 0x00000100,
448 	0x3603, 0xff000ff0, 0x00000100
449 };
450 
451 static const u32 kalindi_golden_spm_registers[] =
452 {
453 	0xc200, 0xe0ffffff, 0xe0000000
454 };
455 
456 static const u32 kalindi_golden_common_registers[] =
457 {
458 	0x31dc, 0xffffffff, 0x00000800,
459 	0x31dd, 0xffffffff, 0x00000800,
460 	0x31e6, 0xffffffff, 0x00007fbf,
461 	0x31e7, 0xffffffff, 0x00007faf
462 };
463 
464 static const u32 kalindi_golden_registers[] =
465 {
466 	0xf000, 0xffffdfff, 0x6e944040,
467 	0x1579, 0xff607fff, 0xfc000100,
468 	0xf088, 0xff000fff, 0x00000100,
469 	0xf089, 0xff000fff, 0x00000100,
470 	0xf080, 0xfffc0fff, 0x00000100,
471 	0x1bb6, 0x00010101, 0x00010000,
472 	0x260c, 0xffffffff, 0x00000000,
473 	0x260d, 0xf00fffff, 0x00000400,
474 	0x16ec, 0x000000f0, 0x00000070,
475 	0x16f0, 0xf0311fff, 0x80300000,
476 	0x263e, 0x73773777, 0x12010001,
477 	0x263f, 0xffffffff, 0x00000010,
478 	0x26df, 0x00ff0000, 0x00fc0000,
479 	0x200c, 0x00001f0f, 0x0000100a,
480 	0xbd2, 0x73773777, 0x12010001,
481 	0x902, 0x000fffff, 0x000c007f,
482 	0x2285, 0xf000003f, 0x00000007,
483 	0x22c9, 0x3fff3fff, 0x00ffcfff,
484 	0xc281, 0x0000ff0f, 0x00000000,
485 	0xa293, 0x07ffffff, 0x06000000,
486 	0x136, 0x00000fff, 0x00000100,
487 	0xf9e, 0x00000001, 0x00000002,
488 	0x31da, 0x00000008, 0x00000008,
489 	0x2300, 0x000000ff, 0x00000003,
490 	0x853e, 0x01ff01ff, 0x00000002,
491 	0x8526, 0x007ff800, 0x00200000,
492 	0x8057, 0xffffffff, 0x00000f40,
493 	0x2231, 0x001f3ae3, 0x00000082,
494 	0x2235, 0x0000001f, 0x00000010,
495 	0xc24d, 0xffffffff, 0x00000000
496 };
497 
498 static const u32 kalindi_mgcg_cgcg_init[] =
499 {
500 	0x3108, 0xffffffff, 0xfffffffc,
501 	0xc200, 0xffffffff, 0xe0000000,
502 	0xf0a8, 0xffffffff, 0x00000100,
503 	0xf082, 0xffffffff, 0x00000100,
504 	0xf0b0, 0xffffffff, 0x00000100,
505 	0xf0b2, 0xffffffff, 0x00000100,
506 	0xf0b1, 0xffffffff, 0x00000100,
507 	0x1579, 0xffffffff, 0x00600100,
508 	0xf0a0, 0xffffffff, 0x00000100,
509 	0xf085, 0xffffffff, 0x06000100,
510 	0xf088, 0xffffffff, 0x00000100,
511 	0xf086, 0xffffffff, 0x06000100,
512 	0xf081, 0xffffffff, 0x00000100,
513 	0xf0b8, 0xffffffff, 0x00000100,
514 	0xf089, 0xffffffff, 0x00000100,
515 	0xf080, 0xffffffff, 0x00000100,
516 	0xf08c, 0xffffffff, 0x00000100,
517 	0xf08d, 0xffffffff, 0x00000100,
518 	0xf094, 0xffffffff, 0x00000100,
519 	0xf095, 0xffffffff, 0x00000100,
520 	0xf096, 0xffffffff, 0x00000100,
521 	0xf097, 0xffffffff, 0x00000100,
522 	0xf098, 0xffffffff, 0x00000100,
523 	0xf09f, 0xffffffff, 0x00000100,
524 	0xf09e, 0xffffffff, 0x00000100,
525 	0xf084, 0xffffffff, 0x06000100,
526 	0xf0a4, 0xffffffff, 0x00000100,
527 	0xf09d, 0xffffffff, 0x00000100,
528 	0xf0ad, 0xffffffff, 0x00000100,
529 	0xf0ac, 0xffffffff, 0x00000100,
530 	0xf09c, 0xffffffff, 0x00000100,
531 	0xc200, 0xffffffff, 0xe0000000,
532 	0xf008, 0xffffffff, 0x00010000,
533 	0xf009, 0xffffffff, 0x00030002,
534 	0xf00a, 0xffffffff, 0x00040007,
535 	0xf00b, 0xffffffff, 0x00060005,
536 	0xf00c, 0xffffffff, 0x00090008,
537 	0xf00d, 0xffffffff, 0x00010000,
538 	0xf00e, 0xffffffff, 0x00030002,
539 	0xf00f, 0xffffffff, 0x00040007,
540 	0xf010, 0xffffffff, 0x00060005,
541 	0xf011, 0xffffffff, 0x00090008,
542 	0xf000, 0xffffffff, 0x96e00200,
543 	0x21c2, 0xffffffff, 0x00900100,
544 	0x3109, 0xffffffff, 0x0020003f,
545 	0xe, 0xffffffff, 0x0140001c,
546 	0xf, 0x000f0000, 0x000f0000,
547 	0x88, 0xffffffff, 0xc060000c,
548 	0x89, 0xc0000fff, 0x00000100,
549 	0x82a, 0xffffffff, 0x00000104,
550 	0x1579, 0xff000fff, 0x00000100,
551 	0xc33, 0xc0000fff, 0x00000104,
552 	0x3079, 0x00000001, 0x00000001,
553 	0x3403, 0xff000ff0, 0x00000100,
554 	0x3603, 0xff000ff0, 0x00000100
555 };
556 
557 static const u32 hawaii_golden_spm_registers[] =
558 {
559 	0xc200, 0xe0ffffff, 0xe0000000
560 };
561 
562 static const u32 hawaii_golden_common_registers[] =
563 {
564 	0xc200, 0xffffffff, 0xe0000000,
565 	0xa0d4, 0xffffffff, 0x3a00161a,
566 	0xa0d5, 0xffffffff, 0x0000002e,
567 	0x2684, 0xffffffff, 0x00018208,
568 	0x263e, 0xffffffff, 0x12011003
569 };
570 
571 static const u32 hawaii_golden_registers[] =
572 {
573 	0xcd5, 0x00000333, 0x00000333,
574 	0x2684, 0x00010000, 0x00058208,
575 	0x260c, 0xffffffff, 0x00000000,
576 	0x260d, 0xf00fffff, 0x00000400,
577 	0x260e, 0x0002021c, 0x00020200,
578 	0x31e, 0x00000080, 0x00000000,
579 	0x16ec, 0x000000f0, 0x00000070,
580 	0x16f0, 0xf0311fff, 0x80300000,
581 	0xd43, 0x00810000, 0x408af000,
582 	0x1c0c, 0x31000111, 0x00000011,
583 	0xbd2, 0x73773777, 0x12010001,
584 	0x848, 0x0000007f, 0x0000001b,
585 	0x877, 0x00007fb6, 0x00002191,
586 	0xd8a, 0x0000003f, 0x0000000a,
587 	0xd8b, 0x0000003f, 0x0000000a,
588 	0xab9, 0x00073ffe, 0x000022a2,
589 	0x903, 0x000007ff, 0x00000000,
590 	0x22fc, 0x00002001, 0x00000001,
591 	0x22c9, 0xffffffff, 0x00ffffff,
592 	0xc281, 0x0000ff0f, 0x00000000,
593 	0xa293, 0x07ffffff, 0x06000000,
594 	0xf9e, 0x00000001, 0x00000002,
595 	0x31da, 0x00000008, 0x00000008,
596 	0x31dc, 0x00000f00, 0x00000800,
597 	0x31dd, 0x00000f00, 0x00000800,
598 	0x31e6, 0x00ffffff, 0x00ff7fbf,
599 	0x31e7, 0x00ffffff, 0x00ff7faf,
600 	0x2300, 0x000000ff, 0x00000800,
601 	0x390, 0x00001fff, 0x00001fff,
602 	0x2418, 0x0000007f, 0x00000020,
603 	0x2542, 0x00010000, 0x00010000,
604 	0x2b80, 0x00100000, 0x000ff07c,
605 	0x2b05, 0x000003ff, 0x0000000f,
606 	0x2b04, 0xffffffff, 0x7564fdec,
607 	0x2b03, 0xffffffff, 0x3120b9a8,
608 	0x2b02, 0x20000000, 0x0f9c0000
609 };
610 
611 static const u32 hawaii_mgcg_cgcg_init[] =
612 {
613 	0x3108, 0xffffffff, 0xfffffffd,
614 	0xc200, 0xffffffff, 0xe0000000,
615 	0xf0a8, 0xffffffff, 0x00000100,
616 	0xf082, 0xffffffff, 0x00000100,
617 	0xf0b0, 0xffffffff, 0x00000100,
618 	0xf0b2, 0xffffffff, 0x00000100,
619 	0xf0b1, 0xffffffff, 0x00000100,
620 	0x1579, 0xffffffff, 0x00200100,
621 	0xf0a0, 0xffffffff, 0x00000100,
622 	0xf085, 0xffffffff, 0x06000100,
623 	0xf088, 0xffffffff, 0x00000100,
624 	0xf086, 0xffffffff, 0x06000100,
625 	0xf081, 0xffffffff, 0x00000100,
626 	0xf0b8, 0xffffffff, 0x00000100,
627 	0xf089, 0xffffffff, 0x00000100,
628 	0xf080, 0xffffffff, 0x00000100,
629 	0xf08c, 0xffffffff, 0x00000100,
630 	0xf08d, 0xffffffff, 0x00000100,
631 	0xf094, 0xffffffff, 0x00000100,
632 	0xf095, 0xffffffff, 0x00000100,
633 	0xf096, 0xffffffff, 0x00000100,
634 	0xf097, 0xffffffff, 0x00000100,
635 	0xf098, 0xffffffff, 0x00000100,
636 	0xf09f, 0xffffffff, 0x00000100,
637 	0xf09e, 0xffffffff, 0x00000100,
638 	0xf084, 0xffffffff, 0x06000100,
639 	0xf0a4, 0xffffffff, 0x00000100,
640 	0xf09d, 0xffffffff, 0x00000100,
641 	0xf0ad, 0xffffffff, 0x00000100,
642 	0xf0ac, 0xffffffff, 0x00000100,
643 	0xf09c, 0xffffffff, 0x00000100,
644 	0xc200, 0xffffffff, 0xe0000000,
645 	0xf008, 0xffffffff, 0x00010000,
646 	0xf009, 0xffffffff, 0x00030002,
647 	0xf00a, 0xffffffff, 0x00040007,
648 	0xf00b, 0xffffffff, 0x00060005,
649 	0xf00c, 0xffffffff, 0x00090008,
650 	0xf00d, 0xffffffff, 0x00010000,
651 	0xf00e, 0xffffffff, 0x00030002,
652 	0xf00f, 0xffffffff, 0x00040007,
653 	0xf010, 0xffffffff, 0x00060005,
654 	0xf011, 0xffffffff, 0x00090008,
655 	0xf012, 0xffffffff, 0x00010000,
656 	0xf013, 0xffffffff, 0x00030002,
657 	0xf014, 0xffffffff, 0x00040007,
658 	0xf015, 0xffffffff, 0x00060005,
659 	0xf016, 0xffffffff, 0x00090008,
660 	0xf017, 0xffffffff, 0x00010000,
661 	0xf018, 0xffffffff, 0x00030002,
662 	0xf019, 0xffffffff, 0x00040007,
663 	0xf01a, 0xffffffff, 0x00060005,
664 	0xf01b, 0xffffffff, 0x00090008,
665 	0xf01c, 0xffffffff, 0x00010000,
666 	0xf01d, 0xffffffff, 0x00030002,
667 	0xf01e, 0xffffffff, 0x00040007,
668 	0xf01f, 0xffffffff, 0x00060005,
669 	0xf020, 0xffffffff, 0x00090008,
670 	0xf021, 0xffffffff, 0x00010000,
671 	0xf022, 0xffffffff, 0x00030002,
672 	0xf023, 0xffffffff, 0x00040007,
673 	0xf024, 0xffffffff, 0x00060005,
674 	0xf025, 0xffffffff, 0x00090008,
675 	0xf026, 0xffffffff, 0x00010000,
676 	0xf027, 0xffffffff, 0x00030002,
677 	0xf028, 0xffffffff, 0x00040007,
678 	0xf029, 0xffffffff, 0x00060005,
679 	0xf02a, 0xffffffff, 0x00090008,
680 	0xf02b, 0xffffffff, 0x00010000,
681 	0xf02c, 0xffffffff, 0x00030002,
682 	0xf02d, 0xffffffff, 0x00040007,
683 	0xf02e, 0xffffffff, 0x00060005,
684 	0xf02f, 0xffffffff, 0x00090008,
685 	0xf030, 0xffffffff, 0x00010000,
686 	0xf031, 0xffffffff, 0x00030002,
687 	0xf032, 0xffffffff, 0x00040007,
688 	0xf033, 0xffffffff, 0x00060005,
689 	0xf034, 0xffffffff, 0x00090008,
690 	0xf035, 0xffffffff, 0x00010000,
691 	0xf036, 0xffffffff, 0x00030002,
692 	0xf037, 0xffffffff, 0x00040007,
693 	0xf038, 0xffffffff, 0x00060005,
694 	0xf039, 0xffffffff, 0x00090008,
695 	0xf03a, 0xffffffff, 0x00010000,
696 	0xf03b, 0xffffffff, 0x00030002,
697 	0xf03c, 0xffffffff, 0x00040007,
698 	0xf03d, 0xffffffff, 0x00060005,
699 	0xf03e, 0xffffffff, 0x00090008,
700 	0x30c6, 0xffffffff, 0x00020200,
701 	0xcd4, 0xffffffff, 0x00000200,
702 	0x570, 0xffffffff, 0x00000400,
703 	0x157a, 0xffffffff, 0x00000000,
704 	0xbd4, 0xffffffff, 0x00000902,
705 	0xf000, 0xffffffff, 0x96940200,
706 	0x21c2, 0xffffffff, 0x00900100,
707 	0x3109, 0xffffffff, 0x0020003f,
708 	0xe, 0xffffffff, 0x0140001c,
709 	0xf, 0x000f0000, 0x000f0000,
710 	0x88, 0xffffffff, 0xc060000c,
711 	0x89, 0xc0000fff, 0x00000100,
712 	0x3e4, 0xffffffff, 0x00000100,
713 	0x3e6, 0x00000101, 0x00000000,
714 	0x82a, 0xffffffff, 0x00000104,
715 	0x1579, 0xff000fff, 0x00000100,
716 	0xc33, 0xc0000fff, 0x00000104,
717 	0x3079, 0x00000001, 0x00000001,
718 	0x3403, 0xff000ff0, 0x00000100,
719 	0x3603, 0xff000ff0, 0x00000100
720 };
721 
722 static const u32 godavari_golden_registers[] =
723 {
724 	0x1579, 0xff607fff, 0xfc000100,
725 	0x1bb6, 0x00010101, 0x00010000,
726 	0x260c, 0xffffffff, 0x00000000,
727 	0x260c0, 0xf00fffff, 0x00000400,
728 	0x184c, 0xffffffff, 0x00010000,
729 	0x16ec, 0x000000f0, 0x00000070,
730 	0x16f0, 0xf0311fff, 0x80300000,
731 	0x263e, 0x73773777, 0x12010001,
732 	0x263f, 0xffffffff, 0x00000010,
733 	0x200c, 0x00001f0f, 0x0000100a,
734 	0xbd2, 0x73773777, 0x12010001,
735 	0x902, 0x000fffff, 0x000c007f,
736 	0x2285, 0xf000003f, 0x00000007,
737 	0x22c9, 0xffffffff, 0x00ff0fff,
738 	0xc281, 0x0000ff0f, 0x00000000,
739 	0xa293, 0x07ffffff, 0x06000000,
740 	0x136, 0x00000fff, 0x00000100,
741 	0x3405, 0x00010000, 0x00810001,
742 	0x3605, 0x00010000, 0x00810001,
743 	0xf9e, 0x00000001, 0x00000002,
744 	0x31da, 0x00000008, 0x00000008,
745 	0x31dc, 0x00000f00, 0x00000800,
746 	0x31dd, 0x00000f00, 0x00000800,
747 	0x31e6, 0x00ffffff, 0x00ff7fbf,
748 	0x31e7, 0x00ffffff, 0x00ff7faf,
749 	0x2300, 0x000000ff, 0x00000001,
750 	0x853e, 0x01ff01ff, 0x00000002,
751 	0x8526, 0x007ff800, 0x00200000,
752 	0x8057, 0xffffffff, 0x00000f40,
753 	0x2231, 0x001f3ae3, 0x00000082,
754 	0x2235, 0x0000001f, 0x00000010,
755 	0xc24d, 0xffffffff, 0x00000000
756 };
757 
758 static void cik_init_golden_registers(struct amdgpu_device *adev)
759 {
760 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
761 	mutex_lock(&adev->grbm_idx_mutex);
762 
763 	switch (adev->asic_type) {
764 	case CHIP_BONAIRE:
765 		amdgpu_device_program_register_sequence(adev,
766 							bonaire_mgcg_cgcg_init,
767 							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
768 		amdgpu_device_program_register_sequence(adev,
769 							bonaire_golden_registers,
770 							ARRAY_SIZE(bonaire_golden_registers));
771 		amdgpu_device_program_register_sequence(adev,
772 							bonaire_golden_common_registers,
773 							ARRAY_SIZE(bonaire_golden_common_registers));
774 		amdgpu_device_program_register_sequence(adev,
775 							bonaire_golden_spm_registers,
776 							ARRAY_SIZE(bonaire_golden_spm_registers));
777 		break;
778 	case CHIP_KABINI:
779 		amdgpu_device_program_register_sequence(adev,
780 							kalindi_mgcg_cgcg_init,
781 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
782 		amdgpu_device_program_register_sequence(adev,
783 							kalindi_golden_registers,
784 							ARRAY_SIZE(kalindi_golden_registers));
785 		amdgpu_device_program_register_sequence(adev,
786 							kalindi_golden_common_registers,
787 							ARRAY_SIZE(kalindi_golden_common_registers));
788 		amdgpu_device_program_register_sequence(adev,
789 							kalindi_golden_spm_registers,
790 							ARRAY_SIZE(kalindi_golden_spm_registers));
791 		break;
792 	case CHIP_MULLINS:
793 		amdgpu_device_program_register_sequence(adev,
794 							kalindi_mgcg_cgcg_init,
795 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
796 		amdgpu_device_program_register_sequence(adev,
797 							godavari_golden_registers,
798 							ARRAY_SIZE(godavari_golden_registers));
799 		amdgpu_device_program_register_sequence(adev,
800 							kalindi_golden_common_registers,
801 							ARRAY_SIZE(kalindi_golden_common_registers));
802 		amdgpu_device_program_register_sequence(adev,
803 							kalindi_golden_spm_registers,
804 							ARRAY_SIZE(kalindi_golden_spm_registers));
805 		break;
806 	case CHIP_KAVERI:
807 		amdgpu_device_program_register_sequence(adev,
808 							spectre_mgcg_cgcg_init,
809 							ARRAY_SIZE(spectre_mgcg_cgcg_init));
810 		amdgpu_device_program_register_sequence(adev,
811 							spectre_golden_registers,
812 							ARRAY_SIZE(spectre_golden_registers));
813 		amdgpu_device_program_register_sequence(adev,
814 							spectre_golden_common_registers,
815 							ARRAY_SIZE(spectre_golden_common_registers));
816 		amdgpu_device_program_register_sequence(adev,
817 							spectre_golden_spm_registers,
818 							ARRAY_SIZE(spectre_golden_spm_registers));
819 		break;
820 	case CHIP_HAWAII:
821 		amdgpu_device_program_register_sequence(adev,
822 							hawaii_mgcg_cgcg_init,
823 							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
824 		amdgpu_device_program_register_sequence(adev,
825 							hawaii_golden_registers,
826 							ARRAY_SIZE(hawaii_golden_registers));
827 		amdgpu_device_program_register_sequence(adev,
828 							hawaii_golden_common_registers,
829 							ARRAY_SIZE(hawaii_golden_common_registers));
830 		amdgpu_device_program_register_sequence(adev,
831 							hawaii_golden_spm_registers,
832 							ARRAY_SIZE(hawaii_golden_spm_registers));
833 		break;
834 	default:
835 		break;
836 	}
837 	mutex_unlock(&adev->grbm_idx_mutex);
838 }
839 
840 /**
841  * cik_get_xclk - get the xclk
842  *
843  * @adev: amdgpu_device pointer
844  *
845  * Returns the reference clock used by the gfx engine
846  * (CIK).
847  */
848 static u32 cik_get_xclk(struct amdgpu_device *adev)
849 {
850 	u32 reference_clock = adev->clock.spll.reference_freq;
851 
852 	if (adev->flags & AMD_IS_APU) {
853 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
854 			return reference_clock / 2;
855 	} else {
856 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
857 			return reference_clock / 4;
858 	}
859 	return reference_clock;
860 }
861 
862 /**
863  * cik_srbm_select - select specific register instances
864  *
865  * @adev: amdgpu_device pointer
866  * @me: selected ME (micro engine)
867  * @pipe: pipe
868  * @queue: queue
869  * @vmid: VMID
870  *
871  * Switches the currently active registers instances.  Some
872  * registers are instanced per VMID, others are instanced per
873  * me/pipe/queue combination.
874  */
875 void cik_srbm_select(struct amdgpu_device *adev,
876 		     u32 me, u32 pipe, u32 queue, u32 vmid)
877 {
878 	u32 srbm_gfx_cntl =
879 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
880 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
881 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
882 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
883 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
884 }
885 
886 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
887 {
888 	uint32_t tmp;
889 
890 	tmp = RREG32(mmCONFIG_CNTL);
891 	if (!state)
892 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
893 	else
894 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
895 	WREG32(mmCONFIG_CNTL, tmp);
896 }
897 
898 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
899 {
900 	u32 bus_cntl;
901 	u32 d1vga_control = 0;
902 	u32 d2vga_control = 0;
903 	u32 vga_render_control = 0;
904 	u32 rom_cntl;
905 	bool r;
906 
907 	bus_cntl = RREG32(mmBUS_CNTL);
908 	if (adev->mode_info.num_crtc) {
909 		d1vga_control = RREG32(mmD1VGA_CONTROL);
910 		d2vga_control = RREG32(mmD2VGA_CONTROL);
911 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
912 	}
913 	rom_cntl = RREG32_SMC(ixROM_CNTL);
914 
915 	/* enable the rom */
916 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
917 	if (adev->mode_info.num_crtc) {
918 		/* Disable VGA mode */
919 		WREG32(mmD1VGA_CONTROL,
920 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
921 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
922 		WREG32(mmD2VGA_CONTROL,
923 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
924 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
925 		WREG32(mmVGA_RENDER_CONTROL,
926 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
927 	}
928 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
929 
930 	r = amdgpu_read_bios(adev);
931 
932 	/* restore regs */
933 	WREG32(mmBUS_CNTL, bus_cntl);
934 	if (adev->mode_info.num_crtc) {
935 		WREG32(mmD1VGA_CONTROL, d1vga_control);
936 		WREG32(mmD2VGA_CONTROL, d2vga_control);
937 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
938 	}
939 	WREG32_SMC(ixROM_CNTL, rom_cntl);
940 	return r;
941 }
942 
943 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
944 				   u8 *bios, u32 length_bytes)
945 {
946 	u32 *dw_ptr;
947 	unsigned long flags;
948 	u32 i, length_dw;
949 
950 	if (bios == NULL)
951 		return false;
952 	if (length_bytes == 0)
953 		return false;
954 	/* APU vbios image is part of sbios image */
955 	if (adev->flags & AMD_IS_APU)
956 		return false;
957 
958 	dw_ptr = (u32 *)bios;
959 	length_dw = ALIGN(length_bytes, 4) / 4;
960 	/* take the smc lock since we are using the smc index */
961 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
962 	/* set rom index to 0 */
963 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
964 	WREG32(mmSMC_IND_DATA_0, 0);
965 	/* set index to data for continous read */
966 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
967 	for (i = 0; i < length_dw; i++)
968 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
969 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
970 
971 	return true;
972 }
973 
974 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
975 	{mmGRBM_STATUS},
976 	{mmGRBM_STATUS2},
977 	{mmGRBM_STATUS_SE0},
978 	{mmGRBM_STATUS_SE1},
979 	{mmGRBM_STATUS_SE2},
980 	{mmGRBM_STATUS_SE3},
981 	{mmSRBM_STATUS},
982 	{mmSRBM_STATUS2},
983 	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
984 	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
985 	{mmCP_STAT},
986 	{mmCP_STALLED_STAT1},
987 	{mmCP_STALLED_STAT2},
988 	{mmCP_STALLED_STAT3},
989 	{mmCP_CPF_BUSY_STAT},
990 	{mmCP_CPF_STALLED_STAT1},
991 	{mmCP_CPF_STATUS},
992 	{mmCP_CPC_BUSY_STAT},
993 	{mmCP_CPC_STALLED_STAT1},
994 	{mmCP_CPC_STATUS},
995 	{mmGB_ADDR_CONFIG},
996 	{mmMC_ARB_RAMCFG},
997 	{mmGB_TILE_MODE0},
998 	{mmGB_TILE_MODE1},
999 	{mmGB_TILE_MODE2},
1000 	{mmGB_TILE_MODE3},
1001 	{mmGB_TILE_MODE4},
1002 	{mmGB_TILE_MODE5},
1003 	{mmGB_TILE_MODE6},
1004 	{mmGB_TILE_MODE7},
1005 	{mmGB_TILE_MODE8},
1006 	{mmGB_TILE_MODE9},
1007 	{mmGB_TILE_MODE10},
1008 	{mmGB_TILE_MODE11},
1009 	{mmGB_TILE_MODE12},
1010 	{mmGB_TILE_MODE13},
1011 	{mmGB_TILE_MODE14},
1012 	{mmGB_TILE_MODE15},
1013 	{mmGB_TILE_MODE16},
1014 	{mmGB_TILE_MODE17},
1015 	{mmGB_TILE_MODE18},
1016 	{mmGB_TILE_MODE19},
1017 	{mmGB_TILE_MODE20},
1018 	{mmGB_TILE_MODE21},
1019 	{mmGB_TILE_MODE22},
1020 	{mmGB_TILE_MODE23},
1021 	{mmGB_TILE_MODE24},
1022 	{mmGB_TILE_MODE25},
1023 	{mmGB_TILE_MODE26},
1024 	{mmGB_TILE_MODE27},
1025 	{mmGB_TILE_MODE28},
1026 	{mmGB_TILE_MODE29},
1027 	{mmGB_TILE_MODE30},
1028 	{mmGB_TILE_MODE31},
1029 	{mmGB_MACROTILE_MODE0},
1030 	{mmGB_MACROTILE_MODE1},
1031 	{mmGB_MACROTILE_MODE2},
1032 	{mmGB_MACROTILE_MODE3},
1033 	{mmGB_MACROTILE_MODE4},
1034 	{mmGB_MACROTILE_MODE5},
1035 	{mmGB_MACROTILE_MODE6},
1036 	{mmGB_MACROTILE_MODE7},
1037 	{mmGB_MACROTILE_MODE8},
1038 	{mmGB_MACROTILE_MODE9},
1039 	{mmGB_MACROTILE_MODE10},
1040 	{mmGB_MACROTILE_MODE11},
1041 	{mmGB_MACROTILE_MODE12},
1042 	{mmGB_MACROTILE_MODE13},
1043 	{mmGB_MACROTILE_MODE14},
1044 	{mmGB_MACROTILE_MODE15},
1045 	{mmCC_RB_BACKEND_DISABLE, true},
1046 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1047 	{mmGB_BACKEND_MAP, false},
1048 	{mmPA_SC_RASTER_CONFIG, true},
1049 	{mmPA_SC_RASTER_CONFIG_1, true},
1050 };
1051 
1052 
1053 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1054 				       bool indexed, u32 se_num,
1055 				       u32 sh_num, u32 reg_offset)
1056 {
1057 	if (indexed) {
1058 		uint32_t val;
1059 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1060 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1061 
1062 		switch (reg_offset) {
1063 		case mmCC_RB_BACKEND_DISABLE:
1064 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1065 		case mmGC_USER_RB_BACKEND_DISABLE:
1066 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1067 		case mmPA_SC_RASTER_CONFIG:
1068 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1069 		case mmPA_SC_RASTER_CONFIG_1:
1070 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1071 		}
1072 
1073 		mutex_lock(&adev->grbm_idx_mutex);
1074 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1075 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1076 
1077 		val = RREG32(reg_offset);
1078 
1079 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1080 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1081 		mutex_unlock(&adev->grbm_idx_mutex);
1082 		return val;
1083 	} else {
1084 		unsigned idx;
1085 
1086 		switch (reg_offset) {
1087 		case mmGB_ADDR_CONFIG:
1088 			return adev->gfx.config.gb_addr_config;
1089 		case mmMC_ARB_RAMCFG:
1090 			return adev->gfx.config.mc_arb_ramcfg;
1091 		case mmGB_TILE_MODE0:
1092 		case mmGB_TILE_MODE1:
1093 		case mmGB_TILE_MODE2:
1094 		case mmGB_TILE_MODE3:
1095 		case mmGB_TILE_MODE4:
1096 		case mmGB_TILE_MODE5:
1097 		case mmGB_TILE_MODE6:
1098 		case mmGB_TILE_MODE7:
1099 		case mmGB_TILE_MODE8:
1100 		case mmGB_TILE_MODE9:
1101 		case mmGB_TILE_MODE10:
1102 		case mmGB_TILE_MODE11:
1103 		case mmGB_TILE_MODE12:
1104 		case mmGB_TILE_MODE13:
1105 		case mmGB_TILE_MODE14:
1106 		case mmGB_TILE_MODE15:
1107 		case mmGB_TILE_MODE16:
1108 		case mmGB_TILE_MODE17:
1109 		case mmGB_TILE_MODE18:
1110 		case mmGB_TILE_MODE19:
1111 		case mmGB_TILE_MODE20:
1112 		case mmGB_TILE_MODE21:
1113 		case mmGB_TILE_MODE22:
1114 		case mmGB_TILE_MODE23:
1115 		case mmGB_TILE_MODE24:
1116 		case mmGB_TILE_MODE25:
1117 		case mmGB_TILE_MODE26:
1118 		case mmGB_TILE_MODE27:
1119 		case mmGB_TILE_MODE28:
1120 		case mmGB_TILE_MODE29:
1121 		case mmGB_TILE_MODE30:
1122 		case mmGB_TILE_MODE31:
1123 			idx = (reg_offset - mmGB_TILE_MODE0);
1124 			return adev->gfx.config.tile_mode_array[idx];
1125 		case mmGB_MACROTILE_MODE0:
1126 		case mmGB_MACROTILE_MODE1:
1127 		case mmGB_MACROTILE_MODE2:
1128 		case mmGB_MACROTILE_MODE3:
1129 		case mmGB_MACROTILE_MODE4:
1130 		case mmGB_MACROTILE_MODE5:
1131 		case mmGB_MACROTILE_MODE6:
1132 		case mmGB_MACROTILE_MODE7:
1133 		case mmGB_MACROTILE_MODE8:
1134 		case mmGB_MACROTILE_MODE9:
1135 		case mmGB_MACROTILE_MODE10:
1136 		case mmGB_MACROTILE_MODE11:
1137 		case mmGB_MACROTILE_MODE12:
1138 		case mmGB_MACROTILE_MODE13:
1139 		case mmGB_MACROTILE_MODE14:
1140 		case mmGB_MACROTILE_MODE15:
1141 			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1142 			return adev->gfx.config.macrotile_mode_array[idx];
1143 		default:
1144 			return RREG32(reg_offset);
1145 		}
1146 	}
1147 }
1148 
1149 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1150 			     u32 sh_num, u32 reg_offset, u32 *value)
1151 {
1152 	uint32_t i;
1153 
1154 	*value = 0;
1155 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1156 		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1157 
1158 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1159 			continue;
1160 
1161 		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1162 						reg_offset);
1163 		return 0;
1164 	}
1165 	return -EINVAL;
1166 }
1167 
1168 struct kv_reset_save_regs {
1169 	u32 gmcon_reng_execute;
1170 	u32 gmcon_misc;
1171 	u32 gmcon_misc3;
1172 };
1173 
1174 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1175 				   struct kv_reset_save_regs *save)
1176 {
1177 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1178 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1179 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1180 
1181 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1182 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1183 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1184 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1185 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1186 }
1187 
1188 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1189 				      struct kv_reset_save_regs *save)
1190 {
1191 	int i;
1192 
1193 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1194 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1195 
1196 	for (i = 0; i < 5; i++)
1197 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1198 
1199 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1200 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1201 
1202 	for (i = 0; i < 5; i++)
1203 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1204 
1205 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1206 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1207 
1208 	for (i = 0; i < 5; i++)
1209 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1210 
1211 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1212 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1213 
1214 	for (i = 0; i < 5; i++)
1215 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1216 
1217 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1218 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1219 
1220 	for (i = 0; i < 5; i++)
1221 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1222 
1223 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1224 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1225 
1226 	for (i = 0; i < 5; i++)
1227 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1228 
1229 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1230 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1231 
1232 	for (i = 0; i < 5; i++)
1233 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1234 
1235 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1236 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1237 
1238 	for (i = 0; i < 5; i++)
1239 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1240 
1241 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1242 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1243 
1244 	for (i = 0; i < 5; i++)
1245 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1246 
1247 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1248 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1249 
1250 	for (i = 0; i < 5; i++)
1251 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1252 
1253 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1254 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1255 
1256 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1257 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1258 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1259 }
1260 
1261 static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1262 {
1263 	struct kv_reset_save_regs kv_save = { 0 };
1264 	u32 i;
1265 	int r = -EINVAL;
1266 
1267 	dev_info(adev->dev, "GPU pci config reset\n");
1268 
1269 	if (adev->flags & AMD_IS_APU)
1270 		kv_save_regs_for_reset(adev, &kv_save);
1271 
1272 	/* disable BM */
1273 	pci_clear_master(adev->pdev);
1274 	/* reset */
1275 	amdgpu_device_pci_config_reset(adev);
1276 
1277 	udelay(100);
1278 
1279 	/* wait for asic to come out of reset */
1280 	for (i = 0; i < adev->usec_timeout; i++) {
1281 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1282 			/* enable BM */
1283 			pci_set_master(adev->pdev);
1284 			adev->has_hw_reset = true;
1285 			r = 0;
1286 			break;
1287 		}
1288 		udelay(1);
1289 	}
1290 
1291 	/* does asic init need to be run first??? */
1292 	if (adev->flags & AMD_IS_APU)
1293 		kv_restore_regs_for_reset(adev, &kv_save);
1294 
1295 	return r;
1296 }
1297 
1298 /**
1299  * cik_asic_pci_config_reset - soft reset GPU
1300  *
1301  * @adev: amdgpu_device pointer
1302  *
1303  * Use PCI Config method to reset the GPU.
1304  *
1305  * Returns 0 for success.
1306  */
1307 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1308 {
1309 	int r;
1310 
1311 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1312 
1313 	r = cik_gpu_pci_config_reset(adev);
1314 
1315 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1316 
1317 	return r;
1318 }
1319 
1320 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1321 {
1322 	switch (adev->asic_type) {
1323 	case CHIP_BONAIRE:
1324 	case CHIP_HAWAII:
1325 		return amdgpu_dpm_is_baco_supported(adev);
1326 	default:
1327 		return false;
1328 	}
1329 }
1330 
1331 static enum amd_reset_method
1332 cik_asic_reset_method(struct amdgpu_device *adev)
1333 {
1334 	bool baco_reset;
1335 
1336 	switch (adev->asic_type) {
1337 	case CHIP_BONAIRE:
1338 	case CHIP_HAWAII:
1339 		/* disable baco reset until it works */
1340 		/* smu7_asic_get_baco_capability(adev, &baco_reset); */
1341 		baco_reset = false;
1342 		break;
1343 	default:
1344 		baco_reset = false;
1345 		break;
1346 	}
1347 
1348 	if (baco_reset)
1349 		return AMD_RESET_METHOD_BACO;
1350 	else
1351 		return AMD_RESET_METHOD_LEGACY;
1352 }
1353 
1354 /**
1355  * cik_asic_reset - soft reset GPU
1356  *
1357  * @adev: amdgpu_device pointer
1358  *
1359  * Look up which blocks are hung and attempt
1360  * to reset them.
1361  * Returns 0 for success.
1362  */
1363 static int cik_asic_reset(struct amdgpu_device *adev)
1364 {
1365 	int r;
1366 
1367 	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1368 		if (!adev->in_suspend)
1369 			amdgpu_inc_vram_lost(adev);
1370 		r = amdgpu_dpm_baco_reset(adev);
1371 	} else {
1372 		r = cik_asic_pci_config_reset(adev);
1373 	}
1374 
1375 	return r;
1376 }
1377 
1378 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1379 {
1380 	return RREG32(mmCONFIG_MEMSIZE);
1381 }
1382 
1383 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1384 			      u32 cntl_reg, u32 status_reg)
1385 {
1386 	int r, i;
1387 	struct atom_clock_dividers dividers;
1388 	uint32_t tmp;
1389 
1390 	r = amdgpu_atombios_get_clock_dividers(adev,
1391 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1392 					       clock, false, &dividers);
1393 	if (r)
1394 		return r;
1395 
1396 	tmp = RREG32_SMC(cntl_reg);
1397 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1398 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1399 	tmp |= dividers.post_divider;
1400 	WREG32_SMC(cntl_reg, tmp);
1401 
1402 	for (i = 0; i < 100; i++) {
1403 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1404 			break;
1405 		mdelay(10);
1406 	}
1407 	if (i == 100)
1408 		return -ETIMEDOUT;
1409 
1410 	return 0;
1411 }
1412 
1413 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1414 {
1415 	int r = 0;
1416 
1417 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1418 	if (r)
1419 		return r;
1420 
1421 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1422 	return r;
1423 }
1424 
1425 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1426 {
1427 	int r, i;
1428 	struct atom_clock_dividers dividers;
1429 	u32 tmp;
1430 
1431 	r = amdgpu_atombios_get_clock_dividers(adev,
1432 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1433 					       ecclk, false, &dividers);
1434 	if (r)
1435 		return r;
1436 
1437 	for (i = 0; i < 100; i++) {
1438 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1439 			break;
1440 		mdelay(10);
1441 	}
1442 	if (i == 100)
1443 		return -ETIMEDOUT;
1444 
1445 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1446 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1447 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1448 	tmp |= dividers.post_divider;
1449 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1450 
1451 	for (i = 0; i < 100; i++) {
1452 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1453 			break;
1454 		mdelay(10);
1455 	}
1456 	if (i == 100)
1457 		return -ETIMEDOUT;
1458 
1459 	return 0;
1460 }
1461 
1462 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1463 {
1464 #ifndef __NetBSD__		/* XXX amdgpu pcie */
1465 	struct pci_dev *root = adev->pdev->bus->self;
1466 	u32 speed_cntl, current_data_rate;
1467 	int i;
1468 	u16 tmp16;
1469 
1470 	if (pci_is_root_bus(adev->pdev->bus))
1471 		return;
1472 
1473 	if (amdgpu_pcie_gen2 == 0)
1474 		return;
1475 
1476 	if (adev->flags & AMD_IS_APU)
1477 		return;
1478 
1479 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1480 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1481 		return;
1482 
1483 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1484 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1485 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1486 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1487 		if (current_data_rate == 2) {
1488 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1489 			return;
1490 		}
1491 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1492 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1493 		if (current_data_rate == 1) {
1494 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1495 			return;
1496 		}
1497 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1498 	}
1499 
1500 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1501 		return;
1502 
1503 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1504 		/* re-try equalization if gen3 is not already enabled */
1505 		if (current_data_rate != 2) {
1506 			u16 bridge_cfg, gpu_cfg;
1507 			u16 bridge_cfg2, gpu_cfg2;
1508 			u32 max_lw, current_lw, tmp;
1509 
1510 			pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1511 						  &bridge_cfg);
1512 			pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1513 						  &gpu_cfg);
1514 
1515 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1516 			pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1517 
1518 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1519 			pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1520 						   tmp16);
1521 
1522 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1523 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1524 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1525 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1526 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1527 
1528 			if (current_lw < max_lw) {
1529 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1530 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1531 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1532 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1533 					tmp |= (max_lw <<
1534 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1535 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1536 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1537 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1538 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1539 				}
1540 			}
1541 
1542 			for (i = 0; i < 10; i++) {
1543 				/* check status */
1544 				pcie_capability_read_word(adev->pdev,
1545 							  PCI_EXP_DEVSTA,
1546 							  &tmp16);
1547 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1548 					break;
1549 
1550 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1551 							  &bridge_cfg);
1552 				pcie_capability_read_word(adev->pdev,
1553 							  PCI_EXP_LNKCTL,
1554 							  &gpu_cfg);
1555 
1556 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1557 							  &bridge_cfg2);
1558 				pcie_capability_read_word(adev->pdev,
1559 							  PCI_EXP_LNKCTL2,
1560 							  &gpu_cfg2);
1561 
1562 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1563 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1564 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1565 
1566 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1567 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1568 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1569 
1570 				msleep(100);
1571 
1572 				/* linkctl */
1573 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1574 							  &tmp16);
1575 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1576 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1577 				pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1578 							   tmp16);
1579 
1580 				pcie_capability_read_word(adev->pdev,
1581 							  PCI_EXP_LNKCTL,
1582 							  &tmp16);
1583 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1584 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1585 				pcie_capability_write_word(adev->pdev,
1586 							   PCI_EXP_LNKCTL,
1587 							   tmp16);
1588 
1589 				/* linkctl2 */
1590 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1591 							  &tmp16);
1592 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1593 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1594 				tmp16 |= (bridge_cfg2 &
1595 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1596 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1597 				pcie_capability_write_word(root,
1598 							   PCI_EXP_LNKCTL2,
1599 							   tmp16);
1600 
1601 				pcie_capability_read_word(adev->pdev,
1602 							  PCI_EXP_LNKCTL2,
1603 							  &tmp16);
1604 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1605 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1606 				tmp16 |= (gpu_cfg2 &
1607 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1608 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1609 				pcie_capability_write_word(adev->pdev,
1610 							   PCI_EXP_LNKCTL2,
1611 							   tmp16);
1612 
1613 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1614 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1615 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1616 			}
1617 		}
1618 	}
1619 
1620 	/* set the link speed */
1621 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1622 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1623 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1624 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1625 
1626 	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1627 	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1628 
1629 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1630 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1631 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1632 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1633 	else
1634 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1635 	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1636 
1637 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1638 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1639 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1640 
1641 	for (i = 0; i < adev->usec_timeout; i++) {
1642 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1643 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1644 			break;
1645 		udelay(1);
1646 	}
1647 #endif
1648 }
1649 
1650 static void cik_program_aspm(struct amdgpu_device *adev)
1651 {
1652 	u32 data, orig;
1653 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1654 	bool disable_clkreq = false;
1655 
1656 	if (amdgpu_aspm == 0)
1657 		return;
1658 
1659 	if (pci_is_root_bus(adev->pdev->bus))
1660 		return;
1661 
1662 	/* XXX double check APUs */
1663 	if (adev->flags & AMD_IS_APU)
1664 		return;
1665 
1666 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1667 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1668 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1669 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1670 	if (orig != data)
1671 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1672 
1673 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1674 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1675 	if (orig != data)
1676 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1677 
1678 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1679 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1680 	if (orig != data)
1681 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1682 
1683 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1684 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1685 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1686 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1687 	if (!disable_l0s)
1688 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1689 
1690 	if (!disable_l1) {
1691 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1692 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1693 		if (orig != data)
1694 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1695 
1696 		if (!disable_plloff_in_l1) {
1697 			bool clk_req_support;
1698 
1699 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1700 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1701 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1702 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1703 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1704 			if (orig != data)
1705 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1706 
1707 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1708 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1709 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1710 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1711 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1712 			if (orig != data)
1713 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1714 
1715 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1716 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1717 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1718 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1719 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1720 			if (orig != data)
1721 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1722 
1723 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1724 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1725 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1726 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1727 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1728 			if (orig != data)
1729 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1730 
1731 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1732 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1733 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1734 			if (orig != data)
1735 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1736 
1737 			if (!disable_clkreq) {
1738 #ifdef __NetBSD__		/* XXX amdgpu pcie */
1739 				clk_req_support = false;
1740 #else
1741 				struct pci_dev *root = adev->pdev->bus->self;
1742 				u32 lnkcap;
1743 
1744 				clk_req_support = false;
1745 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1746 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1747 					clk_req_support = true;
1748 #endif
1749 			} else {
1750 				clk_req_support = false;
1751 			}
1752 
1753 			if (clk_req_support) {
1754 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1755 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1756 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1757 				if (orig != data)
1758 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1759 
1760 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1761 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1762 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1763 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1764 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1765 				if (orig != data)
1766 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1767 
1768 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1769 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1770 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1771 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1772 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1773 				if (orig != data)
1774 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1775 
1776 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1777 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1778 				if (orig != data)
1779 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1780 
1781 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1782 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1783 				if (orig != data)
1784 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1785 
1786 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1787 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1788 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1789 				if (orig != data)
1790 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1791 			}
1792 		}
1793 	} else {
1794 		if (orig != data)
1795 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1796 	}
1797 
1798 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1799 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1800 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1801 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1802 	if (orig != data)
1803 		WREG32_PCIE(ixPCIE_CNTL2, data);
1804 
1805 	if (!disable_l0s) {
1806 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1807 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1808 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1809 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1810 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1811 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1812 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1813 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1814 				if (orig != data)
1815 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1816 			}
1817 		}
1818 	}
1819 }
1820 
1821 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1822 {
1823 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1824 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1825 }
1826 
1827 static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
1828 {
1829 	if (is_virtual_machine()) /* passthrough mode */
1830 		adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1831 }
1832 
1833 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1834 {
1835 	if (!ring || !ring->funcs->emit_wreg) {
1836 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1837 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1838 	} else {
1839 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1840 	}
1841 }
1842 
1843 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1844 			       struct amdgpu_ring *ring)
1845 {
1846 	if (!ring || !ring->funcs->emit_wreg) {
1847 		WREG32(mmHDP_DEBUG0, 1);
1848 		RREG32(mmHDP_DEBUG0);
1849 	} else {
1850 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1851 	}
1852 }
1853 
1854 static bool cik_need_full_reset(struct amdgpu_device *adev)
1855 {
1856 	/* change this when we support soft reset */
1857 	return true;
1858 }
1859 
1860 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1861 			       uint64_t *count1)
1862 {
1863 	uint32_t perfctr = 0;
1864 	uint64_t cnt0_of, cnt1_of;
1865 	int tmp;
1866 
1867 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1868 	 * that may or may not be different from their GPU counterparts
1869 	 */
1870 	if (adev->flags & AMD_IS_APU)
1871 		return;
1872 
1873 	/* Set the 2 events that we wish to watch, defined above */
1874 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1875 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1876 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1877 
1878 	/* Write to enable desired perf counters */
1879 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1880 	/* Zero out and enable the perf counters
1881 	 * Write 0x5:
1882 	 * Bit 0 = Start all counters(1)
1883 	 * Bit 2 = Global counter reset enable(1)
1884 	 */
1885 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1886 
1887 	msleep(1000);
1888 
1889 	/* Load the shadow and disable the perf counters
1890 	 * Write 0x2:
1891 	 * Bit 0 = Stop counters(0)
1892 	 * Bit 1 = Load the shadow counters(1)
1893 	 */
1894 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1895 
1896 	/* Read register values to get any >32bit overflow */
1897 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1898 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1899 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1900 
1901 	/* Get the values and add the overflow */
1902 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1903 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1904 }
1905 
1906 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1907 {
1908 	u32 clock_cntl, pc;
1909 
1910 	if (adev->flags & AMD_IS_APU)
1911 		return false;
1912 
1913 	/* check if the SMC is already running */
1914 	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1915 	pc = RREG32_SMC(ixSMC_PC_C);
1916 	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1917 	    (0x20100 <= pc))
1918 		return true;
1919 
1920 	return false;
1921 }
1922 
1923 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1924 {
1925 	uint64_t nak_r, nak_g;
1926 
1927 	/* Get the number of NAKs received and generated */
1928 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1929 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1930 
1931 	/* Add the total number of NAKs, i.e the number of replays */
1932 	return (nak_r + nak_g);
1933 }
1934 
1935 static const struct amdgpu_asic_funcs cik_asic_funcs =
1936 {
1937 	.read_disabled_bios = &cik_read_disabled_bios,
1938 	.read_bios_from_rom = &cik_read_bios_from_rom,
1939 	.read_register = &cik_read_register,
1940 	.reset = &cik_asic_reset,
1941 	.reset_method = &cik_asic_reset_method,
1942 	.set_vga_state = &cik_vga_set_state,
1943 	.get_xclk = &cik_get_xclk,
1944 	.set_uvd_clocks = &cik_set_uvd_clocks,
1945 	.set_vce_clocks = &cik_set_vce_clocks,
1946 	.get_config_memsize = &cik_get_config_memsize,
1947 	.flush_hdp = &cik_flush_hdp,
1948 	.invalidate_hdp = &cik_invalidate_hdp,
1949 	.need_full_reset = &cik_need_full_reset,
1950 	.init_doorbell_index = &legacy_doorbell_index_init,
1951 	.get_pcie_usage = &cik_get_pcie_usage,
1952 	.need_reset_on_init = &cik_need_reset_on_init,
1953 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1954 	.supports_baco = &cik_asic_supports_baco,
1955 };
1956 
1957 static int cik_common_early_init(void *handle)
1958 {
1959 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1960 
1961 	adev->smc_rreg = &cik_smc_rreg;
1962 	adev->smc_wreg = &cik_smc_wreg;
1963 	adev->pcie_rreg = &cik_pcie_rreg;
1964 	adev->pcie_wreg = &cik_pcie_wreg;
1965 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1966 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1967 	adev->didt_rreg = &cik_didt_rreg;
1968 	adev->didt_wreg = &cik_didt_wreg;
1969 
1970 	adev->asic_funcs = &cik_asic_funcs;
1971 
1972 	adev->rev_id = cik_get_rev_id(adev);
1973 	adev->external_rev_id = 0xFF;
1974 	switch (adev->asic_type) {
1975 	case CHIP_BONAIRE:
1976 		adev->cg_flags =
1977 			AMD_CG_SUPPORT_GFX_MGCG |
1978 			AMD_CG_SUPPORT_GFX_MGLS |
1979 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1980 			AMD_CG_SUPPORT_GFX_CGLS |
1981 			AMD_CG_SUPPORT_GFX_CGTS |
1982 			AMD_CG_SUPPORT_GFX_CGTS_LS |
1983 			AMD_CG_SUPPORT_GFX_CP_LS |
1984 			AMD_CG_SUPPORT_MC_LS |
1985 			AMD_CG_SUPPORT_MC_MGCG |
1986 			AMD_CG_SUPPORT_SDMA_MGCG |
1987 			AMD_CG_SUPPORT_SDMA_LS |
1988 			AMD_CG_SUPPORT_BIF_LS |
1989 			AMD_CG_SUPPORT_VCE_MGCG |
1990 			AMD_CG_SUPPORT_UVD_MGCG |
1991 			AMD_CG_SUPPORT_HDP_LS |
1992 			AMD_CG_SUPPORT_HDP_MGCG;
1993 		adev->pg_flags = 0;
1994 		adev->external_rev_id = adev->rev_id + 0x14;
1995 		break;
1996 	case CHIP_HAWAII:
1997 		adev->cg_flags =
1998 			AMD_CG_SUPPORT_GFX_MGCG |
1999 			AMD_CG_SUPPORT_GFX_MGLS |
2000 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2001 			AMD_CG_SUPPORT_GFX_CGLS |
2002 			AMD_CG_SUPPORT_GFX_CGTS |
2003 			AMD_CG_SUPPORT_GFX_CP_LS |
2004 			AMD_CG_SUPPORT_MC_LS |
2005 			AMD_CG_SUPPORT_MC_MGCG |
2006 			AMD_CG_SUPPORT_SDMA_MGCG |
2007 			AMD_CG_SUPPORT_SDMA_LS |
2008 			AMD_CG_SUPPORT_BIF_LS |
2009 			AMD_CG_SUPPORT_VCE_MGCG |
2010 			AMD_CG_SUPPORT_UVD_MGCG |
2011 			AMD_CG_SUPPORT_HDP_LS |
2012 			AMD_CG_SUPPORT_HDP_MGCG;
2013 		adev->pg_flags = 0;
2014 		adev->external_rev_id = 0x28;
2015 		break;
2016 	case CHIP_KAVERI:
2017 		adev->cg_flags =
2018 			AMD_CG_SUPPORT_GFX_MGCG |
2019 			AMD_CG_SUPPORT_GFX_MGLS |
2020 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2021 			AMD_CG_SUPPORT_GFX_CGLS |
2022 			AMD_CG_SUPPORT_GFX_CGTS |
2023 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2024 			AMD_CG_SUPPORT_GFX_CP_LS |
2025 			AMD_CG_SUPPORT_SDMA_MGCG |
2026 			AMD_CG_SUPPORT_SDMA_LS |
2027 			AMD_CG_SUPPORT_BIF_LS |
2028 			AMD_CG_SUPPORT_VCE_MGCG |
2029 			AMD_CG_SUPPORT_UVD_MGCG |
2030 			AMD_CG_SUPPORT_HDP_LS |
2031 			AMD_CG_SUPPORT_HDP_MGCG;
2032 		adev->pg_flags =
2033 			/*AMD_PG_SUPPORT_GFX_PG |
2034 			  AMD_PG_SUPPORT_GFX_SMG |
2035 			  AMD_PG_SUPPORT_GFX_DMG |*/
2036 			AMD_PG_SUPPORT_UVD |
2037 			AMD_PG_SUPPORT_VCE |
2038 			/*  AMD_PG_SUPPORT_CP |
2039 			  AMD_PG_SUPPORT_GDS |
2040 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2041 			  AMD_PG_SUPPORT_ACP |
2042 			  AMD_PG_SUPPORT_SAMU |*/
2043 			0;
2044 		if (adev->pdev->device == 0x1312 ||
2045 			adev->pdev->device == 0x1316 ||
2046 			adev->pdev->device == 0x1317)
2047 			adev->external_rev_id = 0x41;
2048 		else
2049 			adev->external_rev_id = 0x1;
2050 		break;
2051 	case CHIP_KABINI:
2052 	case CHIP_MULLINS:
2053 		adev->cg_flags =
2054 			AMD_CG_SUPPORT_GFX_MGCG |
2055 			AMD_CG_SUPPORT_GFX_MGLS |
2056 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2057 			AMD_CG_SUPPORT_GFX_CGLS |
2058 			AMD_CG_SUPPORT_GFX_CGTS |
2059 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2060 			AMD_CG_SUPPORT_GFX_CP_LS |
2061 			AMD_CG_SUPPORT_SDMA_MGCG |
2062 			AMD_CG_SUPPORT_SDMA_LS |
2063 			AMD_CG_SUPPORT_BIF_LS |
2064 			AMD_CG_SUPPORT_VCE_MGCG |
2065 			AMD_CG_SUPPORT_UVD_MGCG |
2066 			AMD_CG_SUPPORT_HDP_LS |
2067 			AMD_CG_SUPPORT_HDP_MGCG;
2068 		adev->pg_flags =
2069 			/*AMD_PG_SUPPORT_GFX_PG |
2070 			  AMD_PG_SUPPORT_GFX_SMG | */
2071 			AMD_PG_SUPPORT_UVD |
2072 			/*AMD_PG_SUPPORT_VCE |
2073 			  AMD_PG_SUPPORT_CP |
2074 			  AMD_PG_SUPPORT_GDS |
2075 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2076 			  AMD_PG_SUPPORT_SAMU |*/
2077 			0;
2078 		if (adev->asic_type == CHIP_KABINI) {
2079 			if (adev->rev_id == 0)
2080 				adev->external_rev_id = 0x81;
2081 			else if (adev->rev_id == 1)
2082 				adev->external_rev_id = 0x82;
2083 			else if (adev->rev_id == 2)
2084 				adev->external_rev_id = 0x85;
2085 		} else
2086 			adev->external_rev_id = adev->rev_id + 0xa1;
2087 		break;
2088 	default:
2089 		/* FIXME: not supported yet */
2090 		return -EINVAL;
2091 	}
2092 
2093 	return 0;
2094 }
2095 
2096 static int cik_common_sw_init(void *handle)
2097 {
2098 	return 0;
2099 }
2100 
2101 static int cik_common_sw_fini(void *handle)
2102 {
2103 	return 0;
2104 }
2105 
2106 static int cik_common_hw_init(void *handle)
2107 {
2108 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2109 
2110 	/* move the golden regs per IP block */
2111 	cik_init_golden_registers(adev);
2112 	/* enable pcie gen2/3 link */
2113 	cik_pcie_gen3_enable(adev);
2114 	/* enable aspm */
2115 	cik_program_aspm(adev);
2116 
2117 	return 0;
2118 }
2119 
2120 static int cik_common_hw_fini(void *handle)
2121 {
2122 	return 0;
2123 }
2124 
2125 static int cik_common_suspend(void *handle)
2126 {
2127 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2128 
2129 	return cik_common_hw_fini(adev);
2130 }
2131 
2132 static int cik_common_resume(void *handle)
2133 {
2134 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2135 
2136 	return cik_common_hw_init(adev);
2137 }
2138 
2139 static bool cik_common_is_idle(void *handle)
2140 {
2141 	return true;
2142 }
2143 
2144 static int cik_common_wait_for_idle(void *handle)
2145 {
2146 	return 0;
2147 }
2148 
2149 static int cik_common_soft_reset(void *handle)
2150 {
2151 	/* XXX hard reset?? */
2152 	return 0;
2153 }
2154 
2155 static int cik_common_set_clockgating_state(void *handle,
2156 					    enum amd_clockgating_state state)
2157 {
2158 	return 0;
2159 }
2160 
2161 static int cik_common_set_powergating_state(void *handle,
2162 					    enum amd_powergating_state state)
2163 {
2164 	return 0;
2165 }
2166 
2167 static const struct amd_ip_funcs cik_common_ip_funcs = {
2168 	.name = "cik_common",
2169 	.early_init = cik_common_early_init,
2170 	.late_init = NULL,
2171 	.sw_init = cik_common_sw_init,
2172 	.sw_fini = cik_common_sw_fini,
2173 	.hw_init = cik_common_hw_init,
2174 	.hw_fini = cik_common_hw_fini,
2175 	.suspend = cik_common_suspend,
2176 	.resume = cik_common_resume,
2177 	.is_idle = cik_common_is_idle,
2178 	.wait_for_idle = cik_common_wait_for_idle,
2179 	.soft_reset = cik_common_soft_reset,
2180 	.set_clockgating_state = cik_common_set_clockgating_state,
2181 	.set_powergating_state = cik_common_set_powergating_state,
2182 };
2183 
2184 static const struct amdgpu_ip_block_version cik_common_ip_block =
2185 {
2186 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2187 	.major = 1,
2188 	.minor = 0,
2189 	.rev = 0,
2190 	.funcs = &cik_common_ip_funcs,
2191 };
2192 
2193 int cik_set_ip_blocks(struct amdgpu_device *adev)
2194 {
2195 	cik_detect_hw_virtualization(adev);
2196 
2197 	switch (adev->asic_type) {
2198 	case CHIP_BONAIRE:
2199 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2200 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2201 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2202 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2203 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2204 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2205 		if (adev->enable_virtual_display)
2206 			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2207 #if defined(CONFIG_DRM_AMD_DC)
2208 		else if (amdgpu_device_has_dc_support(adev))
2209 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2210 #endif
2211 		else
2212 			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2213 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2214 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2215 		break;
2216 	case CHIP_HAWAII:
2217 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2218 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2219 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2220 		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2221 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2222 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2223 		if (adev->enable_virtual_display)
2224 			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2225 #if defined(CONFIG_DRM_AMD_DC)
2226 		else if (amdgpu_device_has_dc_support(adev))
2227 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2228 #endif
2229 		else
2230 			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2231 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2232 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2233 		break;
2234 	case CHIP_KAVERI:
2235 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2236 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2237 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2238 		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2239 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2240 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2241 		if (adev->enable_virtual_display)
2242 			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2243 #if defined(CONFIG_DRM_AMD_DC)
2244 		else if (amdgpu_device_has_dc_support(adev))
2245 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2246 #endif
2247 		else
2248 			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2249 
2250 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2251 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2252 		break;
2253 	case CHIP_KABINI:
2254 	case CHIP_MULLINS:
2255 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2256 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2257 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2258 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2259 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2260 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2261 		if (adev->enable_virtual_display)
2262 			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2263 #if defined(CONFIG_DRM_AMD_DC)
2264 		else if (amdgpu_device_has_dc_support(adev))
2265 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2266 #endif
2267 		else
2268 			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2269 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2270 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2271 		break;
2272 	default:
2273 		/* FIXME: not supported yet */
2274 		return -EINVAL;
2275 	}
2276 	return 0;
2277 }
2278