1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 #include "hdp/hdp_4_0_offset.h" 42 43 #include "soc15_common.h" 44 #include "clearstate_gfx9.h" 45 #include "v9_structs.h" 46 47 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 48 49 #include "amdgpu_ras.h" 50 51 #include "gfx_v9_4.h" 52 #include "gfx_v9_0.h" 53 54 #include "asic_reg/pwr/pwr_10_0_offset.h" 55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 56 57 #define GFX9_NUM_GFX_RINGS 1 58 #define GFX9_MEC_HPD_SIZE 4096 59 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 60 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 61 62 #define mmGCEA_PROBE_MAP 0x070c 63 #define mmGCEA_PROBE_MAP_BASE_IDX 0 64 65 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 66 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 71 72 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 73 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 78 79 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 80 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 85 86 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 87 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 92 93 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 94 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 100 101 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 102 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 107 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 108 109 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 110 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin"); 118 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 119 120 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 125 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 126 127 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 128 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 129 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 130 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 131 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 132 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 133 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 134 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 135 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 136 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 137 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 138 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 139 140 enum ta_ras_gfx_subblock { 141 /*CPC*/ 142 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 143 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 144 TA_RAS_BLOCK__GFX_CPC_UCODE, 145 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 146 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 147 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 148 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 151 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 152 /* CPF*/ 153 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 154 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 155 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 156 TA_RAS_BLOCK__GFX_CPF_TAG, 157 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 158 /* CPG*/ 159 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 160 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 161 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 162 TA_RAS_BLOCK__GFX_CPG_TAG, 163 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 164 /* GDS*/ 165 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 166 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 167 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 168 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 169 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 170 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 171 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 172 /* SPI*/ 173 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 174 /* SQ*/ 175 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 176 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 177 TA_RAS_BLOCK__GFX_SQ_LDS_D, 178 TA_RAS_BLOCK__GFX_SQ_LDS_I, 179 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 180 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 181 /* SQC (3 ranges)*/ 182 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 183 /* SQC range 0*/ 184 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 185 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 186 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 187 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 188 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 189 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 190 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 191 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 192 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 193 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 195 /* SQC range 1*/ 196 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 197 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 198 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 199 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 201 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 202 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 203 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 204 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 207 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 209 /* SQC range 2*/ 210 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 211 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 212 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 213 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 215 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 217 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 218 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 221 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 223 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 224 /* TA*/ 225 TA_RAS_BLOCK__GFX_TA_INDEX_START, 226 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 227 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 228 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 229 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 230 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 231 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 232 /* TCA*/ 233 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 234 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 235 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 236 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 237 /* TCC (5 sub-ranges)*/ 238 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 239 /* TCC range 0*/ 240 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 241 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 242 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 243 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 245 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 246 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 247 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 248 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 249 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 250 /* TCC range 1*/ 251 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 252 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 253 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 254 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 255 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 256 /* TCC range 2*/ 257 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 258 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 259 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 260 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 261 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 262 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 263 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 264 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 265 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 266 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 267 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 268 /* TCC range 3*/ 269 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 270 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 271 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 272 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 274 /* TCC range 4*/ 275 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 276 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 277 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 278 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 279 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 280 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 281 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 282 /* TCI*/ 283 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 284 /* TCP*/ 285 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 286 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 287 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 288 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 289 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 290 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 291 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 292 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 293 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 294 /* TD*/ 295 TA_RAS_BLOCK__GFX_TD_INDEX_START, 296 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 297 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 298 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 299 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 300 /* EA (3 sub-ranges)*/ 301 TA_RAS_BLOCK__GFX_EA_INDEX_START, 302 /* EA range 0*/ 303 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 304 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 305 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 306 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 307 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 308 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 309 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 310 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 311 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 312 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 313 /* EA range 1*/ 314 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 315 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 316 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 317 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 318 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 319 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 320 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 321 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 322 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 323 /* EA range 2*/ 324 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 325 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 326 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 327 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 328 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 329 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 330 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 331 /* UTC VM L2 bank*/ 332 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 333 /* UTC VM walker*/ 334 TA_RAS_BLOCK__UTC_VML2_WALKER, 335 /* UTC ATC L2 2MB cache*/ 336 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 337 /* UTC ATC L2 4KB cache*/ 338 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 339 TA_RAS_BLOCK__GFX_MAX 340 }; 341 342 struct ras_gfx_subblock { 343 unsigned char *name; 344 int ta_subblock; 345 int hw_supported_error_type; 346 int sw_supported_error_type; 347 }; 348 349 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 350 [AMDGPU_RAS_BLOCK__##subblock] = { \ 351 #subblock, \ 352 TA_RAS_BLOCK__##subblock, \ 353 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 354 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 355 } 356 357 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 358 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 359 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 360 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 361 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 362 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 369 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 371 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 372 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 374 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 375 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 377 0), 378 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 380 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 382 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 386 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 388 0), 389 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 390 0, 0), 391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 392 0), 393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 394 0, 0), 395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 396 0), 397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 398 1), 399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 400 0, 0, 0), 401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 402 0), 403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 404 0), 405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 406 0), 407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 408 0), 409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 410 0), 411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 412 0, 0), 413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 414 0), 415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 416 0), 417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 418 0, 0, 0), 419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 420 0), 421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 422 0), 423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 424 0), 425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 426 0), 427 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 428 0), 429 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 430 0, 0), 431 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 432 0), 433 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 434 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 435 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 438 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 442 1), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 444 1), 445 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 446 1), 447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 448 0), 449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 450 0), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 463 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 466 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 468 0, 0), 469 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 470 0), 471 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 505 }; 506 507 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 508 { 509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 529 }; 530 531 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 532 { 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 551 }; 552 553 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 554 { 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 566 }; 567 568 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 569 { 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 594 }; 595 596 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 597 { 598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 600 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 605 }; 606 607 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 608 { 609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 628 }; 629 630 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 631 { 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 644 }; 645 646 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 647 { 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 651 }; 652 653 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 654 { 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 671 }; 672 673 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 674 { 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 688 }; 689 690 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 691 { 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 703 }; 704 705 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 706 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 707 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 708 }; 709 710 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 711 { 712 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 713 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 714 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 715 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 720 }; 721 722 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 723 { 724 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 725 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 726 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 727 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 732 }; 733 734 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v) 735 { 736 static void *scratch_reg0; 737 static void *scratch_reg1; 738 static void *scratch_reg2; 739 static void *scratch_reg3; 740 static void *spare_int; 741 static uint32_t grbm_cntl; 742 static uint32_t grbm_idx; 743 744 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 745 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 746 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 747 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 748 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 749 750 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 751 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 752 753 if (amdgpu_sriov_runtime(adev)) { 754 pr_err("shouldn't call rlcg write register during runtime\n"); 755 return; 756 } 757 758 if (offset == grbm_cntl || offset == grbm_idx) { 759 if (offset == grbm_cntl) 760 writel(v, scratch_reg2); 761 else if (offset == grbm_idx) 762 writel(v, scratch_reg3); 763 764 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 765 } else { 766 uint32_t i = 0; 767 uint32_t retries = 50000; 768 769 writel(v, scratch_reg0); 770 writel(offset | 0x80000000, scratch_reg1); 771 writel(1, spare_int); 772 for (i = 0; i < retries; i++) { 773 u32 tmp; 774 775 tmp = readl(scratch_reg1); 776 if (!(tmp & 0x80000000)) 777 break; 778 779 udelay(10); 780 } 781 if (i >= retries) 782 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 783 } 784 785 } 786 787 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 788 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 789 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 790 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 791 792 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 793 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 794 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 795 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 796 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 797 struct amdgpu_cu_info *cu_info); 798 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 799 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 800 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 801 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 802 void *ras_error_status); 803 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 804 void *inject_if); 805 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 806 807 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 808 uint64_t queue_mask) 809 { 810 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 811 amdgpu_ring_write(kiq_ring, 812 PACKET3_SET_RESOURCES_VMID_MASK(0) | 813 /* vmid_mask:0* queue_type:0 (KIQ) */ 814 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 815 amdgpu_ring_write(kiq_ring, 816 lower_32_bits(queue_mask)); /* queue mask lo */ 817 amdgpu_ring_write(kiq_ring, 818 upper_32_bits(queue_mask)); /* queue mask hi */ 819 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 820 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 821 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 822 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 823 } 824 825 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 826 struct amdgpu_ring *ring) 827 { 828 struct amdgpu_device *adev = kiq_ring->adev; 829 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 830 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 831 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 832 833 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 834 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 835 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 836 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 837 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 838 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 839 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 840 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 841 /*queue_type: normal compute queue */ 842 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 843 /* alloc format: all_on_one_pipe */ 844 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 845 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 846 /* num_queues: must be 1 */ 847 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 848 amdgpu_ring_write(kiq_ring, 849 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 850 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 851 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 852 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 853 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 854 } 855 856 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 857 struct amdgpu_ring *ring, 858 enum amdgpu_unmap_queues_action action, 859 u64 gpu_addr, u64 seq) 860 { 861 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 862 863 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 864 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 865 PACKET3_UNMAP_QUEUES_ACTION(action) | 866 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 867 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 868 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 869 amdgpu_ring_write(kiq_ring, 870 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 871 872 if (action == PREEMPT_QUEUES_NO_UNMAP) { 873 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 874 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 875 amdgpu_ring_write(kiq_ring, seq); 876 } else { 877 amdgpu_ring_write(kiq_ring, 0); 878 amdgpu_ring_write(kiq_ring, 0); 879 amdgpu_ring_write(kiq_ring, 0); 880 } 881 } 882 883 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 884 struct amdgpu_ring *ring, 885 u64 addr, 886 u64 seq) 887 { 888 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 889 890 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 891 amdgpu_ring_write(kiq_ring, 892 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 893 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 894 PACKET3_QUERY_STATUS_COMMAND(2)); 895 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 896 amdgpu_ring_write(kiq_ring, 897 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 898 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 899 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 900 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 901 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 902 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 903 } 904 905 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 906 uint16_t pasid, uint32_t flush_type, 907 bool all_hub) 908 { 909 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 910 amdgpu_ring_write(kiq_ring, 911 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 912 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 913 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 914 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 915 } 916 917 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 918 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 919 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 920 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 921 .kiq_query_status = gfx_v9_0_kiq_query_status, 922 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 923 .set_resources_size = 8, 924 .map_queues_size = 7, 925 .unmap_queues_size = 6, 926 .query_status_size = 7, 927 .invalidate_tlbs_size = 2, 928 }; 929 930 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 931 { 932 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 933 } 934 935 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 936 { 937 switch (adev->asic_type) { 938 case CHIP_VEGA10: 939 soc15_program_register_sequence(adev, 940 golden_settings_gc_9_0, 941 ARRAY_SIZE(golden_settings_gc_9_0)); 942 soc15_program_register_sequence(adev, 943 golden_settings_gc_9_0_vg10, 944 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 945 break; 946 case CHIP_VEGA12: 947 soc15_program_register_sequence(adev, 948 golden_settings_gc_9_2_1, 949 ARRAY_SIZE(golden_settings_gc_9_2_1)); 950 soc15_program_register_sequence(adev, 951 golden_settings_gc_9_2_1_vg12, 952 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 953 break; 954 case CHIP_VEGA20: 955 soc15_program_register_sequence(adev, 956 golden_settings_gc_9_0, 957 ARRAY_SIZE(golden_settings_gc_9_0)); 958 soc15_program_register_sequence(adev, 959 golden_settings_gc_9_0_vg20, 960 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 961 break; 962 case CHIP_ARCTURUS: 963 soc15_program_register_sequence(adev, 964 golden_settings_gc_9_4_1_arct, 965 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 966 break; 967 case CHIP_RAVEN: 968 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 969 ARRAY_SIZE(golden_settings_gc_9_1)); 970 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 971 soc15_program_register_sequence(adev, 972 golden_settings_gc_9_1_rv2, 973 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 974 else 975 soc15_program_register_sequence(adev, 976 golden_settings_gc_9_1_rv1, 977 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 978 break; 979 case CHIP_RENOIR: 980 soc15_program_register_sequence(adev, 981 golden_settings_gc_9_1_rn, 982 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 983 return; /* for renoir, don't need common goldensetting */ 984 default: 985 break; 986 } 987 988 if (adev->asic_type != CHIP_ARCTURUS) 989 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 990 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 991 } 992 993 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 994 { 995 adev->gfx.scratch.num_reg = 8; 996 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 997 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 998 } 999 1000 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1001 bool wc, uint32_t reg, uint32_t val) 1002 { 1003 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1004 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1005 WRITE_DATA_DST_SEL(0) | 1006 (wc ? WR_CONFIRM : 0)); 1007 amdgpu_ring_write(ring, reg); 1008 amdgpu_ring_write(ring, 0); 1009 amdgpu_ring_write(ring, val); 1010 } 1011 1012 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1013 int mem_space, int opt, uint32_t addr0, 1014 uint32_t addr1, uint32_t ref, uint32_t mask, 1015 uint32_t inv) 1016 { 1017 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1018 amdgpu_ring_write(ring, 1019 /* memory (1) or register (0) */ 1020 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1021 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1022 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1023 WAIT_REG_MEM_ENGINE(eng_sel))); 1024 1025 if (mem_space) 1026 BUG_ON(addr0 & 0x3); /* Dword align */ 1027 amdgpu_ring_write(ring, addr0); 1028 amdgpu_ring_write(ring, addr1); 1029 amdgpu_ring_write(ring, ref); 1030 amdgpu_ring_write(ring, mask); 1031 amdgpu_ring_write(ring, inv); /* poll interval */ 1032 } 1033 1034 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1035 { 1036 struct amdgpu_device *adev = ring->adev; 1037 uint32_t scratch; 1038 uint32_t tmp = 0; 1039 unsigned i; 1040 int r; 1041 1042 r = amdgpu_gfx_scratch_get(adev, &scratch); 1043 if (r) 1044 return r; 1045 1046 WREG32(scratch, 0xCAFEDEAD); 1047 r = amdgpu_ring_alloc(ring, 3); 1048 if (r) 1049 goto error_free_scratch; 1050 1051 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1052 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1053 amdgpu_ring_write(ring, 0xDEADBEEF); 1054 amdgpu_ring_commit(ring); 1055 1056 for (i = 0; i < adev->usec_timeout; i++) { 1057 tmp = RREG32(scratch); 1058 if (tmp == 0xDEADBEEF) 1059 break; 1060 udelay(1); 1061 } 1062 1063 if (i >= adev->usec_timeout) 1064 r = -ETIMEDOUT; 1065 1066 error_free_scratch: 1067 amdgpu_gfx_scratch_free(adev, scratch); 1068 return r; 1069 } 1070 1071 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1072 { 1073 struct amdgpu_device *adev = ring->adev; 1074 struct amdgpu_ib ib; 1075 struct dma_fence *f = NULL; 1076 1077 unsigned index; 1078 uint64_t gpu_addr; 1079 uint32_t tmp; 1080 long r; 1081 1082 r = amdgpu_device_wb_get(adev, &index); 1083 if (r) 1084 return r; 1085 1086 gpu_addr = adev->wb.gpu_addr + (index * 4); 1087 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1088 memset(&ib, 0, sizeof(ib)); 1089 r = amdgpu_ib_get(adev, NULL, 16, 1090 AMDGPU_IB_POOL_DIRECT, &ib); 1091 if (r) 1092 goto err1; 1093 1094 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1095 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1096 ib.ptr[2] = lower_32_bits(gpu_addr); 1097 ib.ptr[3] = upper_32_bits(gpu_addr); 1098 ib.ptr[4] = 0xDEADBEEF; 1099 ib.length_dw = 5; 1100 1101 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1102 if (r) 1103 goto err2; 1104 1105 r = dma_fence_wait_timeout(f, false, timeout); 1106 if (r == 0) { 1107 r = -ETIMEDOUT; 1108 goto err2; 1109 } else if (r < 0) { 1110 goto err2; 1111 } 1112 1113 tmp = adev->wb.wb[index]; 1114 if (tmp == 0xDEADBEEF) 1115 r = 0; 1116 else 1117 r = -EINVAL; 1118 1119 err2: 1120 amdgpu_ib_free(adev, &ib, NULL); 1121 dma_fence_put(f); 1122 err1: 1123 amdgpu_device_wb_free(adev, index); 1124 return r; 1125 } 1126 1127 1128 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1129 { 1130 release_firmware(adev->gfx.pfp_fw); 1131 adev->gfx.pfp_fw = NULL; 1132 release_firmware(adev->gfx.me_fw); 1133 adev->gfx.me_fw = NULL; 1134 release_firmware(adev->gfx.ce_fw); 1135 adev->gfx.ce_fw = NULL; 1136 release_firmware(adev->gfx.rlc_fw); 1137 adev->gfx.rlc_fw = NULL; 1138 release_firmware(adev->gfx.mec_fw); 1139 adev->gfx.mec_fw = NULL; 1140 release_firmware(adev->gfx.mec2_fw); 1141 adev->gfx.mec2_fw = NULL; 1142 1143 kfree(adev->gfx.rlc.register_list_format); 1144 } 1145 1146 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1147 { 1148 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1149 1150 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1151 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1152 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1153 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1154 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1155 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1156 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1157 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1158 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1159 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1160 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1161 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1162 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1163 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1164 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1165 } 1166 1167 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1168 { 1169 adev->gfx.me_fw_write_wait = false; 1170 adev->gfx.mec_fw_write_wait = false; 1171 1172 if ((adev->asic_type != CHIP_ARCTURUS) && 1173 ((adev->gfx.mec_fw_version < 0x000001a5) || 1174 (adev->gfx.mec_feature_version < 46) || 1175 (adev->gfx.pfp_fw_version < 0x000000b7) || 1176 (adev->gfx.pfp_feature_version < 46))) 1177 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1178 1179 switch (adev->asic_type) { 1180 case CHIP_VEGA10: 1181 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1182 (adev->gfx.me_feature_version >= 42) && 1183 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1184 (adev->gfx.pfp_feature_version >= 42)) 1185 adev->gfx.me_fw_write_wait = true; 1186 1187 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1188 (adev->gfx.mec_feature_version >= 42)) 1189 adev->gfx.mec_fw_write_wait = true; 1190 break; 1191 case CHIP_VEGA12: 1192 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1193 (adev->gfx.me_feature_version >= 44) && 1194 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1195 (adev->gfx.pfp_feature_version >= 44)) 1196 adev->gfx.me_fw_write_wait = true; 1197 1198 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1199 (adev->gfx.mec_feature_version >= 44)) 1200 adev->gfx.mec_fw_write_wait = true; 1201 break; 1202 case CHIP_VEGA20: 1203 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1204 (adev->gfx.me_feature_version >= 44) && 1205 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1206 (adev->gfx.pfp_feature_version >= 44)) 1207 adev->gfx.me_fw_write_wait = true; 1208 1209 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1210 (adev->gfx.mec_feature_version >= 44)) 1211 adev->gfx.mec_fw_write_wait = true; 1212 break; 1213 case CHIP_RAVEN: 1214 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1215 (adev->gfx.me_feature_version >= 42) && 1216 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1217 (adev->gfx.pfp_feature_version >= 42)) 1218 adev->gfx.me_fw_write_wait = true; 1219 1220 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1221 (adev->gfx.mec_feature_version >= 42)) 1222 adev->gfx.mec_fw_write_wait = true; 1223 break; 1224 default: 1225 adev->gfx.me_fw_write_wait = true; 1226 adev->gfx.mec_fw_write_wait = true; 1227 break; 1228 } 1229 } 1230 1231 struct amdgpu_gfxoff_quirk { 1232 u16 chip_vendor; 1233 u16 chip_device; 1234 u16 subsys_vendor; 1235 u16 subsys_device; 1236 u8 revision; 1237 }; 1238 1239 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1240 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1241 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1242 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1243 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1244 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1245 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1246 { 0, 0, 0, 0, 0 }, 1247 }; 1248 1249 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1250 { 1251 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1252 1253 while (p && p->chip_device != 0) { 1254 if (pdev->vendor == p->chip_vendor && 1255 pdev->device == p->chip_device && 1256 pdev->subsystem_vendor == p->subsys_vendor && 1257 pdev->subsystem_device == p->subsys_device && 1258 pdev->revision == p->revision) { 1259 return true; 1260 } 1261 ++p; 1262 } 1263 return false; 1264 } 1265 1266 static bool is_raven_kicker(struct amdgpu_device *adev) 1267 { 1268 if (adev->pm.fw_version >= 0x41e2b) 1269 return true; 1270 else 1271 return false; 1272 } 1273 1274 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) 1275 { 1276 if ((adev->asic_type == CHIP_RENOIR) && 1277 (adev->gfx.me_fw_version >= 0x000000a5) && 1278 (adev->gfx.me_feature_version >= 52)) 1279 return true; 1280 else 1281 return false; 1282 } 1283 1284 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1285 { 1286 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1287 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1288 1289 switch (adev->asic_type) { 1290 case CHIP_VEGA10: 1291 case CHIP_VEGA12: 1292 case CHIP_VEGA20: 1293 break; 1294 case CHIP_RAVEN: 1295 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1296 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1297 ((!is_raven_kicker(adev) && 1298 adev->gfx.rlc_fw_version < 531) || 1299 (adev->gfx.rlc_feature_version < 1) || 1300 !adev->gfx.rlc.is_rlc_v2_1)) 1301 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1302 1303 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1304 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1305 AMD_PG_SUPPORT_CP | 1306 AMD_PG_SUPPORT_RLC_SMU_HS; 1307 break; 1308 case CHIP_RENOIR: 1309 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1310 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1311 AMD_PG_SUPPORT_CP | 1312 AMD_PG_SUPPORT_RLC_SMU_HS; 1313 break; 1314 default: 1315 break; 1316 } 1317 } 1318 1319 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1320 const char *chip_name) 1321 { 1322 char fw_name[30]; 1323 int err; 1324 struct amdgpu_firmware_info *info = NULL; 1325 const struct common_firmware_header *header = NULL; 1326 const struct gfx_firmware_header_v1_0 *cp_hdr; 1327 1328 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1329 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1330 if (err) 1331 goto out; 1332 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1333 if (err) 1334 goto out; 1335 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1336 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1337 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1338 1339 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1340 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1341 if (err) 1342 goto out; 1343 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1344 if (err) 1345 goto out; 1346 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1347 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1348 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1349 1350 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1351 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1352 if (err) 1353 goto out; 1354 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1355 if (err) 1356 goto out; 1357 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1358 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1359 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1360 1361 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1362 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1363 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1364 info->fw = adev->gfx.pfp_fw; 1365 header = (const struct common_firmware_header *)info->fw->data; 1366 adev->firmware.fw_size += 1367 roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1368 1369 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1370 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1371 info->fw = adev->gfx.me_fw; 1372 header = (const struct common_firmware_header *)info->fw->data; 1373 adev->firmware.fw_size += 1374 roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1375 1376 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1377 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1378 info->fw = adev->gfx.ce_fw; 1379 header = (const struct common_firmware_header *)info->fw->data; 1380 adev->firmware.fw_size += 1381 roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1382 } 1383 1384 out: 1385 if (err) { 1386 dev_err(adev->dev, 1387 "gfx9: Failed to load firmware \"%s\"\n", 1388 fw_name); 1389 release_firmware(adev->gfx.pfp_fw); 1390 adev->gfx.pfp_fw = NULL; 1391 release_firmware(adev->gfx.me_fw); 1392 adev->gfx.me_fw = NULL; 1393 release_firmware(adev->gfx.ce_fw); 1394 adev->gfx.ce_fw = NULL; 1395 } 1396 return err; 1397 } 1398 1399 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1400 const char *chip_name) 1401 { 1402 char fw_name[30]; 1403 int err; 1404 struct amdgpu_firmware_info *info = NULL; 1405 const struct common_firmware_header *header = NULL; 1406 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1407 unsigned int *tmp = NULL; 1408 unsigned int i = 0; 1409 uint16_t version_major; 1410 uint16_t version_minor; 1411 uint32_t smu_version; 1412 1413 /* 1414 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1415 * instead of picasso_rlc.bin. 1416 * Judgment method: 1417 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1418 * or revision >= 0xD8 && revision <= 0xDF 1419 * otherwise is PCO FP5 1420 */ 1421 if (!strcmp(chip_name, "picasso") && 1422 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1423 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1424 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1425 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1426 (smu_version >= 0x41e2b)) 1427 /** 1428 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1429 */ 1430 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1431 else 1432 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1433 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1434 if (err) 1435 goto out; 1436 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1437 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1438 1439 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1440 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1441 if (version_major == 2 && version_minor == 1) 1442 adev->gfx.rlc.is_rlc_v2_1 = true; 1443 1444 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1445 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1446 adev->gfx.rlc.save_and_restore_offset = 1447 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1448 adev->gfx.rlc.clear_state_descriptor_offset = 1449 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1450 adev->gfx.rlc.avail_scratch_ram_locations = 1451 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1452 adev->gfx.rlc.reg_restore_list_size = 1453 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1454 adev->gfx.rlc.reg_list_format_start = 1455 le32_to_cpu(rlc_hdr->reg_list_format_start); 1456 adev->gfx.rlc.reg_list_format_separate_start = 1457 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1458 adev->gfx.rlc.starting_offsets_start = 1459 le32_to_cpu(rlc_hdr->starting_offsets_start); 1460 adev->gfx.rlc.reg_list_format_size_bytes = 1461 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1462 adev->gfx.rlc.reg_list_size_bytes = 1463 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1464 adev->gfx.rlc.register_list_format = 1465 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1466 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1467 if (!adev->gfx.rlc.register_list_format) { 1468 err = -ENOMEM; 1469 goto out; 1470 } 1471 1472 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1473 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1474 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1475 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1476 1477 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1478 1479 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1480 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1481 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1482 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1483 1484 if (adev->gfx.rlc.is_rlc_v2_1) 1485 gfx_v9_0_init_rlc_ext_microcode(adev); 1486 1487 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1488 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1489 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1490 info->fw = adev->gfx.rlc_fw; 1491 header = (const struct common_firmware_header *)info->fw->data; 1492 adev->firmware.fw_size += 1493 roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1494 1495 if (adev->gfx.rlc.is_rlc_v2_1 && 1496 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1497 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1498 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1499 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1500 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1501 info->fw = adev->gfx.rlc_fw; 1502 adev->firmware.fw_size += 1503 roundup2(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1504 1505 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1506 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1507 info->fw = adev->gfx.rlc_fw; 1508 adev->firmware.fw_size += 1509 roundup2(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1510 1511 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1512 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1513 info->fw = adev->gfx.rlc_fw; 1514 adev->firmware.fw_size += 1515 roundup2(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1516 } 1517 } 1518 1519 out: 1520 if (err) { 1521 dev_err(adev->dev, 1522 "gfx9: Failed to load firmware \"%s\"\n", 1523 fw_name); 1524 release_firmware(adev->gfx.rlc_fw); 1525 adev->gfx.rlc_fw = NULL; 1526 } 1527 return err; 1528 } 1529 1530 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1531 const char *chip_name) 1532 { 1533 char fw_name[30]; 1534 int err; 1535 struct amdgpu_firmware_info *info = NULL; 1536 const struct common_firmware_header *header = NULL; 1537 const struct gfx_firmware_header_v1_0 *cp_hdr; 1538 1539 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1540 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1541 if (err) 1542 goto out; 1543 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1544 if (err) 1545 goto out; 1546 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1547 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1548 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1549 1550 1551 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1552 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1553 if (!err) { 1554 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1555 if (err) 1556 goto out; 1557 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1558 adev->gfx.mec2_fw->data; 1559 adev->gfx.mec2_fw_version = 1560 le32_to_cpu(cp_hdr->header.ucode_version); 1561 adev->gfx.mec2_feature_version = 1562 le32_to_cpu(cp_hdr->ucode_feature_version); 1563 } else { 1564 err = 0; 1565 adev->gfx.mec2_fw = NULL; 1566 } 1567 1568 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1569 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1570 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1571 info->fw = adev->gfx.mec_fw; 1572 header = (const struct common_firmware_header *)info->fw->data; 1573 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1574 adev->firmware.fw_size += 1575 roundup2(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1576 1577 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1578 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1579 info->fw = adev->gfx.mec_fw; 1580 adev->firmware.fw_size += 1581 roundup2(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1582 1583 if (adev->gfx.mec2_fw) { 1584 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1585 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1586 info->fw = adev->gfx.mec2_fw; 1587 header = (const struct common_firmware_header *)info->fw->data; 1588 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1589 adev->firmware.fw_size += 1590 roundup2(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1591 1592 /* TODO: Determine if MEC2 JT FW loading can be removed 1593 for all GFX V9 asic and above */ 1594 if (adev->asic_type != CHIP_ARCTURUS && 1595 adev->asic_type != CHIP_RENOIR) { 1596 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1597 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1598 info->fw = adev->gfx.mec2_fw; 1599 adev->firmware.fw_size += 1600 roundup2(le32_to_cpu(cp_hdr->jt_size) * 4, 1601 PAGE_SIZE); 1602 } 1603 } 1604 } 1605 1606 out: 1607 gfx_v9_0_check_if_need_gfxoff(adev); 1608 gfx_v9_0_check_fw_write_wait(adev); 1609 if (err) { 1610 dev_err(adev->dev, 1611 "gfx9: Failed to load firmware \"%s\"\n", 1612 fw_name); 1613 release_firmware(adev->gfx.mec_fw); 1614 adev->gfx.mec_fw = NULL; 1615 release_firmware(adev->gfx.mec2_fw); 1616 adev->gfx.mec2_fw = NULL; 1617 } 1618 return err; 1619 } 1620 1621 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1622 { 1623 const char *chip_name; 1624 int r; 1625 1626 DRM_DEBUG("\n"); 1627 1628 switch (adev->asic_type) { 1629 case CHIP_VEGA10: 1630 chip_name = "vega10"; 1631 break; 1632 case CHIP_VEGA12: 1633 chip_name = "vega12"; 1634 break; 1635 case CHIP_VEGA20: 1636 chip_name = "vega20"; 1637 break; 1638 case CHIP_RAVEN: 1639 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1640 chip_name = "raven2"; 1641 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1642 chip_name = "picasso"; 1643 else 1644 chip_name = "raven"; 1645 break; 1646 case CHIP_ARCTURUS: 1647 chip_name = "arcturus"; 1648 break; 1649 case CHIP_RENOIR: 1650 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1651 chip_name = "renoir"; 1652 else 1653 chip_name = "green_sardine"; 1654 break; 1655 default: 1656 BUG(); 1657 } 1658 1659 /* No CPG in Arcturus */ 1660 if (adev->asic_type != CHIP_ARCTURUS) { 1661 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1662 if (r) 1663 return r; 1664 } 1665 1666 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1667 if (r) 1668 return r; 1669 1670 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1671 if (r) 1672 return r; 1673 1674 return r; 1675 } 1676 1677 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1678 { 1679 u32 count = 0; 1680 const struct cs_section_def *sect = NULL; 1681 const struct cs_extent_def *ext = NULL; 1682 1683 /* begin clear state */ 1684 count += 2; 1685 /* context control state */ 1686 count += 3; 1687 1688 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1689 for (ext = sect->section; ext->extent != NULL; ++ext) { 1690 if (sect->id == SECT_CONTEXT) 1691 count += 2 + ext->reg_count; 1692 else 1693 return 0; 1694 } 1695 } 1696 1697 /* end clear state */ 1698 count += 2; 1699 /* clear state */ 1700 count += 2; 1701 1702 return count; 1703 } 1704 1705 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1706 volatile u32 *buffer) 1707 { 1708 u32 count = 0, i; 1709 const struct cs_section_def *sect = NULL; 1710 const struct cs_extent_def *ext = NULL; 1711 1712 if (adev->gfx.rlc.cs_data == NULL) 1713 return; 1714 if (buffer == NULL) 1715 return; 1716 1717 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1718 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1719 1720 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1721 buffer[count++] = cpu_to_le32(0x80000000); 1722 buffer[count++] = cpu_to_le32(0x80000000); 1723 1724 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1725 for (ext = sect->section; ext->extent != NULL; ++ext) { 1726 if (sect->id == SECT_CONTEXT) { 1727 buffer[count++] = 1728 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1729 buffer[count++] = cpu_to_le32(ext->reg_index - 1730 PACKET3_SET_CONTEXT_REG_START); 1731 for (i = 0; i < ext->reg_count; i++) 1732 buffer[count++] = cpu_to_le32(ext->extent[i]); 1733 } else { 1734 return; 1735 } 1736 } 1737 } 1738 1739 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1740 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1741 1742 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1743 buffer[count++] = cpu_to_le32(0); 1744 } 1745 1746 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1747 { 1748 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1749 uint32_t pg_always_on_cu_num = 2; 1750 uint32_t always_on_cu_num; 1751 uint32_t i, j, k; 1752 uint32_t mask, cu_bitmap, counter; 1753 1754 if (adev->flags & AMD_IS_APU) 1755 always_on_cu_num = 4; 1756 else if (adev->asic_type == CHIP_VEGA12) 1757 always_on_cu_num = 8; 1758 else 1759 always_on_cu_num = 12; 1760 1761 mutex_lock(&adev->grbm_idx_mutex); 1762 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1763 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1764 mask = 1; 1765 cu_bitmap = 0; 1766 counter = 0; 1767 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1768 1769 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1770 if (cu_info->bitmap[i][j] & mask) { 1771 if (counter == pg_always_on_cu_num) 1772 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1773 if (counter < always_on_cu_num) 1774 cu_bitmap |= mask; 1775 else 1776 break; 1777 counter++; 1778 } 1779 mask <<= 1; 1780 } 1781 1782 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1783 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1784 } 1785 } 1786 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1787 mutex_unlock(&adev->grbm_idx_mutex); 1788 } 1789 1790 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1791 { 1792 uint32_t data; 1793 1794 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1795 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1796 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1797 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1798 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1799 1800 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1801 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1802 1803 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1804 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1805 1806 mutex_lock(&adev->grbm_idx_mutex); 1807 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1808 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1809 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1810 1811 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1812 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1813 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1814 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1815 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1816 1817 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1818 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1819 data &= 0x0000FFFF; 1820 data |= 0x00C00000; 1821 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1822 1823 /* 1824 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1825 * programmed in gfx_v9_0_init_always_on_cu_mask() 1826 */ 1827 1828 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1829 * but used for RLC_LB_CNTL configuration */ 1830 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1831 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1832 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1833 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1834 mutex_unlock(&adev->grbm_idx_mutex); 1835 1836 gfx_v9_0_init_always_on_cu_mask(adev); 1837 } 1838 1839 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1840 { 1841 uint32_t data; 1842 1843 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1844 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1845 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1846 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1847 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1848 1849 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1850 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1851 1852 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1853 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1854 1855 mutex_lock(&adev->grbm_idx_mutex); 1856 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1857 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1858 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1859 1860 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1861 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1862 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1863 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1864 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1865 1866 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1867 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1868 data &= 0x0000FFFF; 1869 data |= 0x00C00000; 1870 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1871 1872 /* 1873 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1874 * programmed in gfx_v9_0_init_always_on_cu_mask() 1875 */ 1876 1877 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1878 * but used for RLC_LB_CNTL configuration */ 1879 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1880 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1881 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1882 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1883 mutex_unlock(&adev->grbm_idx_mutex); 1884 1885 gfx_v9_0_init_always_on_cu_mask(adev); 1886 } 1887 1888 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1889 { 1890 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1891 } 1892 1893 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1894 { 1895 return 5; 1896 } 1897 1898 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1899 { 1900 const struct cs_section_def *cs_data; 1901 int r; 1902 1903 adev->gfx.rlc.cs_data = gfx9_cs_data; 1904 1905 cs_data = adev->gfx.rlc.cs_data; 1906 1907 if (cs_data) { 1908 /* init clear state block */ 1909 r = amdgpu_gfx_rlc_init_csb(adev); 1910 if (r) 1911 return r; 1912 } 1913 1914 if (adev->flags & AMD_IS_APU) { 1915 /* TODO: double check the cp_table_size for RV */ 1916 adev->gfx.rlc.cp_table_size = roundup2(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1917 r = amdgpu_gfx_rlc_init_cpt(adev); 1918 if (r) 1919 return r; 1920 } 1921 1922 switch (adev->asic_type) { 1923 case CHIP_RAVEN: 1924 gfx_v9_0_init_lbpw(adev); 1925 break; 1926 case CHIP_VEGA20: 1927 gfx_v9_4_init_lbpw(adev); 1928 break; 1929 default: 1930 break; 1931 } 1932 1933 /* init spm vmid with 0xf */ 1934 if (adev->gfx.rlc.funcs->update_spm_vmid) 1935 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1936 1937 return 0; 1938 } 1939 1940 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1941 { 1942 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1943 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1944 } 1945 1946 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1947 { 1948 int r; 1949 u32 *hpd; 1950 const __le32 *fw_data; 1951 unsigned fw_size; 1952 u32 *fw; 1953 size_t mec_hpd_size; 1954 1955 const struct gfx_firmware_header_v1_0 *mec_hdr; 1956 1957 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1958 1959 /* take ownership of the relevant compute queues */ 1960 amdgpu_gfx_compute_queue_acquire(adev); 1961 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1962 if (mec_hpd_size) { 1963 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1964 AMDGPU_GEM_DOMAIN_VRAM, 1965 &adev->gfx.mec.hpd_eop_obj, 1966 &adev->gfx.mec.hpd_eop_gpu_addr, 1967 (void **)&hpd); 1968 if (r) { 1969 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1970 gfx_v9_0_mec_fini(adev); 1971 return r; 1972 } 1973 1974 memset(hpd, 0, mec_hpd_size); 1975 1976 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1977 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1978 } 1979 1980 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1981 1982 fw_data = (const __le32 *) 1983 (adev->gfx.mec_fw->data + 1984 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1985 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1986 1987 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1988 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1989 &adev->gfx.mec.mec_fw_obj, 1990 &adev->gfx.mec.mec_fw_gpu_addr, 1991 (void **)&fw); 1992 if (r) { 1993 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1994 gfx_v9_0_mec_fini(adev); 1995 return r; 1996 } 1997 1998 memcpy(fw, fw_data, fw_size); 1999 2000 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2001 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2002 2003 return 0; 2004 } 2005 2006 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 2007 { 2008 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2009 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2010 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2011 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2012 (SQ_IND_INDEX__FORCE_READ_MASK)); 2013 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2014 } 2015 2016 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2017 uint32_t wave, uint32_t thread, 2018 uint32_t regno, uint32_t num, uint32_t *out) 2019 { 2020 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2021 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2022 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2023 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2024 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2025 (SQ_IND_INDEX__FORCE_READ_MASK) | 2026 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2027 while (num--) 2028 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2029 } 2030 2031 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2032 { 2033 /* type 1 wave data */ 2034 dst[(*no_fields)++] = 1; 2035 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2036 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2037 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2038 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2039 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2040 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2041 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2042 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2043 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2044 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2045 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2046 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2047 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2048 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2049 } 2050 2051 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2052 uint32_t wave, uint32_t start, 2053 uint32_t size, uint32_t *dst) 2054 { 2055 wave_read_regs( 2056 adev, simd, wave, 0, 2057 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2058 } 2059 2060 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2061 uint32_t wave, uint32_t thread, 2062 uint32_t start, uint32_t size, 2063 uint32_t *dst) 2064 { 2065 wave_read_regs( 2066 adev, simd, wave, thread, 2067 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2068 } 2069 2070 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2071 u32 me, u32 pipe, u32 q, u32 vm) 2072 { 2073 soc15_grbm_select(adev, me, pipe, q, vm); 2074 } 2075 2076 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2077 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2078 .select_se_sh = &gfx_v9_0_select_se_sh, 2079 .read_wave_data = &gfx_v9_0_read_wave_data, 2080 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2081 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2082 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2083 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2084 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2085 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2086 }; 2087 2088 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { 2089 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2090 .select_se_sh = &gfx_v9_0_select_se_sh, 2091 .read_wave_data = &gfx_v9_0_read_wave_data, 2092 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2093 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2094 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2095 .ras_error_inject = &gfx_v9_4_ras_error_inject, 2096 .query_ras_error_count = &gfx_v9_4_query_ras_error_count, 2097 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count, 2098 .query_ras_error_status = &gfx_v9_4_query_ras_error_status, 2099 }; 2100 2101 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2102 { 2103 u32 gb_addr_config; 2104 int err; 2105 2106 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2107 2108 switch (adev->asic_type) { 2109 case CHIP_VEGA10: 2110 adev->gfx.config.max_hw_contexts = 8; 2111 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2112 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2113 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2114 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2115 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2116 break; 2117 case CHIP_VEGA12: 2118 adev->gfx.config.max_hw_contexts = 8; 2119 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2120 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2121 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2122 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2123 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2124 DRM_INFO("fix gfx.config for vega12\n"); 2125 break; 2126 case CHIP_VEGA20: 2127 adev->gfx.config.max_hw_contexts = 8; 2128 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2129 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2130 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2131 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2132 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2133 gb_addr_config &= ~0xf3e777ff; 2134 gb_addr_config |= 0x22014042; 2135 /* check vbios table if gpu info is not available */ 2136 err = amdgpu_atomfirmware_get_gfx_info(adev); 2137 if (err) 2138 return err; 2139 break; 2140 case CHIP_RAVEN: 2141 adev->gfx.config.max_hw_contexts = 8; 2142 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2143 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2144 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2145 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2146 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2147 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2148 else 2149 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2150 break; 2151 case CHIP_ARCTURUS: 2152 adev->gfx.funcs = &gfx_v9_4_gfx_funcs; 2153 adev->gfx.config.max_hw_contexts = 8; 2154 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2155 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2156 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2157 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2158 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2159 gb_addr_config &= ~0xf3e777ff; 2160 gb_addr_config |= 0x22014042; 2161 break; 2162 case CHIP_RENOIR: 2163 adev->gfx.config.max_hw_contexts = 8; 2164 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2165 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2166 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2167 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2168 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2169 gb_addr_config &= ~0xf3e777ff; 2170 gb_addr_config |= 0x22010042; 2171 break; 2172 default: 2173 BUG(); 2174 break; 2175 } 2176 2177 adev->gfx.config.gb_addr_config = gb_addr_config; 2178 2179 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2180 REG_GET_FIELD( 2181 adev->gfx.config.gb_addr_config, 2182 GB_ADDR_CONFIG, 2183 NUM_PIPES); 2184 2185 adev->gfx.config.max_tile_pipes = 2186 adev->gfx.config.gb_addr_config_fields.num_pipes; 2187 2188 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2189 REG_GET_FIELD( 2190 adev->gfx.config.gb_addr_config, 2191 GB_ADDR_CONFIG, 2192 NUM_BANKS); 2193 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2194 REG_GET_FIELD( 2195 adev->gfx.config.gb_addr_config, 2196 GB_ADDR_CONFIG, 2197 MAX_COMPRESSED_FRAGS); 2198 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2199 REG_GET_FIELD( 2200 adev->gfx.config.gb_addr_config, 2201 GB_ADDR_CONFIG, 2202 NUM_RB_PER_SE); 2203 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2204 REG_GET_FIELD( 2205 adev->gfx.config.gb_addr_config, 2206 GB_ADDR_CONFIG, 2207 NUM_SHADER_ENGINES); 2208 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2209 REG_GET_FIELD( 2210 adev->gfx.config.gb_addr_config, 2211 GB_ADDR_CONFIG, 2212 PIPE_INTERLEAVE_SIZE)); 2213 2214 return 0; 2215 } 2216 2217 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2218 int mec, int pipe, int queue) 2219 { 2220 unsigned irq_type; 2221 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2222 unsigned int hw_prio; 2223 2224 ring = &adev->gfx.compute_ring[ring_id]; 2225 2226 /* mec0 is me1 */ 2227 ring->me = mec + 1; 2228 ring->pipe = pipe; 2229 ring->queue = queue; 2230 2231 ring->ring_obj = NULL; 2232 ring->use_doorbell = true; 2233 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2234 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2235 + (ring_id * GFX9_MEC_HPD_SIZE); 2236 snprintf(ring->name, sizeof(ring->name), "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2237 2238 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2239 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2240 + ring->pipe; 2241 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, 2242 ring->queue) ? 2243 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2244 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2245 return amdgpu_ring_init(adev, ring, 1024, 2246 &adev->gfx.eop_irq, irq_type, hw_prio); 2247 } 2248 2249 static int gfx_v9_0_sw_init(void *handle) 2250 { 2251 int i, j, k, r, ring_id; 2252 struct amdgpu_ring *ring; 2253 struct amdgpu_kiq *kiq; 2254 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2255 2256 switch (adev->asic_type) { 2257 case CHIP_VEGA10: 2258 case CHIP_VEGA12: 2259 case CHIP_VEGA20: 2260 case CHIP_RAVEN: 2261 case CHIP_ARCTURUS: 2262 case CHIP_RENOIR: 2263 adev->gfx.mec.num_mec = 2; 2264 break; 2265 default: 2266 adev->gfx.mec.num_mec = 1; 2267 break; 2268 } 2269 2270 adev->gfx.mec.num_pipe_per_mec = 4; 2271 adev->gfx.mec.num_queue_per_pipe = 8; 2272 2273 /* EOP Event */ 2274 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2275 if (r) 2276 return r; 2277 2278 /* Privileged reg */ 2279 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2280 &adev->gfx.priv_reg_irq); 2281 if (r) 2282 return r; 2283 2284 /* Privileged inst */ 2285 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2286 &adev->gfx.priv_inst_irq); 2287 if (r) 2288 return r; 2289 2290 /* ECC error */ 2291 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2292 &adev->gfx.cp_ecc_error_irq); 2293 if (r) 2294 return r; 2295 2296 /* FUE error */ 2297 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2298 &adev->gfx.cp_ecc_error_irq); 2299 if (r) 2300 return r; 2301 2302 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2303 2304 gfx_v9_0_scratch_init(adev); 2305 2306 r = gfx_v9_0_init_microcode(adev); 2307 if (r) { 2308 DRM_ERROR("Failed to load gfx firmware!\n"); 2309 return r; 2310 } 2311 2312 r = adev->gfx.rlc.funcs->init(adev); 2313 if (r) { 2314 DRM_ERROR("Failed to init rlc BOs!\n"); 2315 return r; 2316 } 2317 2318 r = gfx_v9_0_mec_init(adev); 2319 if (r) { 2320 DRM_ERROR("Failed to init MEC BOs!\n"); 2321 return r; 2322 } 2323 2324 /* set up the gfx ring */ 2325 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2326 ring = &adev->gfx.gfx_ring[i]; 2327 ring->ring_obj = NULL; 2328 if (!i) 2329 snprintf(ring->name, sizeof(ring->name), "gfx"); 2330 else 2331 snprintf(ring->name, sizeof(ring->name), "gfx_%d", i); 2332 ring->use_doorbell = true; 2333 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2334 r = amdgpu_ring_init(adev, ring, 1024, 2335 &adev->gfx.eop_irq, 2336 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2337 AMDGPU_RING_PRIO_DEFAULT); 2338 if (r) 2339 return r; 2340 } 2341 2342 /* set up the compute queues - allocate horizontally across pipes */ 2343 ring_id = 0; 2344 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2345 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2346 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2347 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2348 continue; 2349 2350 r = gfx_v9_0_compute_ring_init(adev, 2351 ring_id, 2352 i, k, j); 2353 if (r) 2354 return r; 2355 2356 ring_id++; 2357 } 2358 } 2359 } 2360 2361 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2362 if (r) { 2363 DRM_ERROR("Failed to init KIQ BOs!\n"); 2364 return r; 2365 } 2366 2367 kiq = &adev->gfx.kiq; 2368 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2369 if (r) 2370 return r; 2371 2372 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2373 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2374 if (r) 2375 return r; 2376 2377 adev->gfx.ce_ram_size = 0x8000; 2378 2379 r = gfx_v9_0_gpu_early_init(adev); 2380 if (r) 2381 return r; 2382 2383 return 0; 2384 } 2385 2386 2387 static int gfx_v9_0_sw_fini(void *handle) 2388 { 2389 int i; 2390 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2391 2392 amdgpu_gfx_ras_fini(adev); 2393 2394 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2395 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2396 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2397 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2398 2399 amdgpu_gfx_mqd_sw_fini(adev); 2400 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2401 amdgpu_gfx_kiq_fini(adev); 2402 2403 gfx_v9_0_mec_fini(adev); 2404 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2405 if (adev->flags & AMD_IS_APU) { 2406 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2407 &adev->gfx.rlc.cp_table_gpu_addr, 2408 (void **)&adev->gfx.rlc.cp_table_ptr); 2409 } 2410 gfx_v9_0_free_microcode(adev); 2411 2412 return 0; 2413 } 2414 2415 2416 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2417 { 2418 /* TODO */ 2419 } 2420 2421 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2422 u32 instance) 2423 { 2424 u32 data; 2425 2426 if (instance == 0xffffffff) 2427 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2428 else 2429 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2430 2431 if (se_num == 0xffffffff) 2432 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2433 else 2434 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2435 2436 if (sh_num == 0xffffffff) 2437 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2438 else 2439 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2440 2441 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2442 } 2443 2444 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2445 { 2446 u32 data, mask; 2447 2448 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2449 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2450 2451 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2452 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2453 2454 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2455 adev->gfx.config.max_sh_per_se); 2456 2457 return (~data) & mask; 2458 } 2459 2460 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2461 { 2462 int i, j; 2463 u32 data; 2464 u32 active_rbs = 0; 2465 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2466 adev->gfx.config.max_sh_per_se; 2467 2468 mutex_lock(&adev->grbm_idx_mutex); 2469 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2470 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2471 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2472 data = gfx_v9_0_get_rb_active_bitmap(adev); 2473 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2474 rb_bitmap_width_per_sh); 2475 } 2476 } 2477 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2478 mutex_unlock(&adev->grbm_idx_mutex); 2479 2480 adev->gfx.config.backend_enable_mask = active_rbs; 2481 adev->gfx.config.num_rbs = hweight32(active_rbs); 2482 } 2483 2484 #define DEFAULT_SH_MEM_BASES (0x6000) 2485 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2486 { 2487 int i; 2488 uint32_t sh_mem_config; 2489 uint32_t sh_mem_bases; 2490 2491 /* 2492 * Configure apertures: 2493 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2494 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2495 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2496 */ 2497 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2498 2499 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2500 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2501 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2502 2503 mutex_lock(&adev->srbm_mutex); 2504 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2505 soc15_grbm_select(adev, 0, 0, 0, i); 2506 /* CP and shaders */ 2507 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2508 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2509 } 2510 soc15_grbm_select(adev, 0, 0, 0, 0); 2511 mutex_unlock(&adev->srbm_mutex); 2512 2513 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2514 acccess. These should be enabled by FW for target VMIDs. */ 2515 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2516 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2517 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2518 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2519 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2520 } 2521 } 2522 2523 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2524 { 2525 int vmid; 2526 2527 /* 2528 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2529 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2530 * the driver can enable them for graphics. VMID0 should maintain 2531 * access so that HWS firmware can save/restore entries. 2532 */ 2533 for (vmid = 1; vmid < 16; vmid++) { 2534 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2535 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2536 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2537 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2538 } 2539 } 2540 2541 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2542 { 2543 uint32_t tmp; 2544 2545 switch (adev->asic_type) { 2546 case CHIP_ARCTURUS: 2547 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2548 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2549 DISABLE_BARRIER_WAITCNT, 1); 2550 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2551 break; 2552 default: 2553 break; 2554 } 2555 } 2556 2557 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2558 { 2559 u32 tmp; 2560 int i; 2561 2562 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2563 2564 gfx_v9_0_tiling_mode_table_init(adev); 2565 2566 gfx_v9_0_setup_rb(adev); 2567 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2568 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2569 2570 /* XXX SH_MEM regs */ 2571 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2572 mutex_lock(&adev->srbm_mutex); 2573 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2574 soc15_grbm_select(adev, 0, 0, 0, i); 2575 /* CP and shaders */ 2576 if (i == 0) { 2577 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2578 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2579 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2580 !!adev->gmc.noretry); 2581 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2582 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2583 } else { 2584 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2585 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2586 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2587 !!adev->gmc.noretry); 2588 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2589 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2590 (adev->gmc.private_aperture_start >> 48)); 2591 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2592 (adev->gmc.shared_aperture_start >> 48)); 2593 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2594 } 2595 } 2596 soc15_grbm_select(adev, 0, 0, 0, 0); 2597 2598 mutex_unlock(&adev->srbm_mutex); 2599 2600 gfx_v9_0_init_compute_vmid(adev); 2601 gfx_v9_0_init_gds_vmid(adev); 2602 gfx_v9_0_init_sq_config(adev); 2603 } 2604 2605 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2606 { 2607 u32 i, j, k; 2608 u32 mask; 2609 2610 mutex_lock(&adev->grbm_idx_mutex); 2611 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2612 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2613 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2614 for (k = 0; k < adev->usec_timeout; k++) { 2615 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2616 break; 2617 udelay(1); 2618 } 2619 if (k == adev->usec_timeout) { 2620 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2621 0xffffffff, 0xffffffff); 2622 mutex_unlock(&adev->grbm_idx_mutex); 2623 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2624 i, j); 2625 return; 2626 } 2627 } 2628 } 2629 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2630 mutex_unlock(&adev->grbm_idx_mutex); 2631 2632 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2633 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2634 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2635 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2636 for (k = 0; k < adev->usec_timeout; k++) { 2637 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2638 break; 2639 udelay(1); 2640 } 2641 } 2642 2643 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2644 bool enable) 2645 { 2646 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2647 2648 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2649 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2650 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2651 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2652 2653 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2654 } 2655 2656 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2657 { 2658 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2659 /* csib */ 2660 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2661 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2662 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2663 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2664 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2665 adev->gfx.rlc.clear_state_size); 2666 } 2667 2668 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2669 int indirect_offset, 2670 int list_size, 2671 int *unique_indirect_regs, 2672 int unique_indirect_reg_count, 2673 int *indirect_start_offsets, 2674 int *indirect_start_offsets_count, 2675 int max_start_offsets_count) 2676 { 2677 int idx; 2678 2679 for (; indirect_offset < list_size; indirect_offset++) { 2680 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2681 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2682 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2683 2684 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2685 indirect_offset += 2; 2686 2687 /* look for the matching indice */ 2688 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2689 if (unique_indirect_regs[idx] == 2690 register_list_format[indirect_offset] || 2691 !unique_indirect_regs[idx]) 2692 break; 2693 } 2694 2695 BUG_ON(idx >= unique_indirect_reg_count); 2696 2697 if (!unique_indirect_regs[idx]) 2698 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2699 2700 indirect_offset++; 2701 } 2702 } 2703 } 2704 2705 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2706 { 2707 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2708 int unique_indirect_reg_count = 0; 2709 2710 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2711 int indirect_start_offsets_count = 0; 2712 2713 int list_size = 0; 2714 int i = 0, j = 0; 2715 u32 tmp = 0; 2716 2717 u32 *register_list_format = 2718 kmemdup(adev->gfx.rlc.register_list_format, 2719 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2720 if (!register_list_format) 2721 return -ENOMEM; 2722 2723 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2724 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2725 gfx_v9_1_parse_ind_reg_list(register_list_format, 2726 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2727 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2728 unique_indirect_regs, 2729 unique_indirect_reg_count, 2730 indirect_start_offsets, 2731 &indirect_start_offsets_count, 2732 ARRAY_SIZE(indirect_start_offsets)); 2733 2734 /* enable auto inc in case it is disabled */ 2735 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2736 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2737 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2738 2739 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2740 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2741 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2742 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2743 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2744 adev->gfx.rlc.register_restore[i]); 2745 2746 /* load indirect register */ 2747 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2748 adev->gfx.rlc.reg_list_format_start); 2749 2750 /* direct register portion */ 2751 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2752 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2753 register_list_format[i]); 2754 2755 /* indirect register portion */ 2756 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2757 if (register_list_format[i] == 0xFFFFFFFF) { 2758 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2759 continue; 2760 } 2761 2762 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2763 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2764 2765 for (j = 0; j < unique_indirect_reg_count; j++) { 2766 if (register_list_format[i] == unique_indirect_regs[j]) { 2767 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2768 break; 2769 } 2770 } 2771 2772 BUG_ON(j >= unique_indirect_reg_count); 2773 2774 i++; 2775 } 2776 2777 /* set save/restore list size */ 2778 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2779 list_size = list_size >> 1; 2780 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2781 adev->gfx.rlc.reg_restore_list_size); 2782 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2783 2784 /* write the starting offsets to RLC scratch ram */ 2785 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2786 adev->gfx.rlc.starting_offsets_start); 2787 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2788 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2789 indirect_start_offsets[i]); 2790 2791 /* load unique indirect regs*/ 2792 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2793 if (unique_indirect_regs[i] != 0) { 2794 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2795 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2796 unique_indirect_regs[i] & 0x3FFFF); 2797 2798 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2799 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2800 unique_indirect_regs[i] >> 20); 2801 } 2802 } 2803 2804 kfree(register_list_format); 2805 return 0; 2806 } 2807 2808 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2809 { 2810 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2811 } 2812 2813 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2814 bool enable) 2815 { 2816 uint32_t data = 0; 2817 uint32_t default_data = 0; 2818 2819 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2820 if (enable) { 2821 /* enable GFXIP control over CGPG */ 2822 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2823 if(default_data != data) 2824 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2825 2826 /* update status */ 2827 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2828 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2829 if(default_data != data) 2830 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2831 } else { 2832 /* restore GFXIP control over GCPG */ 2833 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2834 if(default_data != data) 2835 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2836 } 2837 } 2838 2839 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2840 { 2841 uint32_t data = 0; 2842 2843 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2844 AMD_PG_SUPPORT_GFX_SMG | 2845 AMD_PG_SUPPORT_GFX_DMG)) { 2846 /* init IDLE_POLL_COUNT = 60 */ 2847 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2848 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2849 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2850 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2851 2852 /* init RLC PG Delay */ 2853 data = 0; 2854 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2855 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2856 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2857 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2858 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2859 2860 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2861 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2862 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2863 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2864 2865 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2866 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2867 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2868 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2869 2870 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2871 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2872 2873 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2874 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2875 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2876 if (adev->asic_type != CHIP_RENOIR) 2877 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2878 } 2879 } 2880 2881 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2882 bool enable) 2883 { 2884 uint32_t data = 0; 2885 uint32_t default_data = 0; 2886 2887 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2888 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2889 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2890 enable ? 1 : 0); 2891 if (default_data != data) 2892 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2893 } 2894 2895 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2896 bool enable) 2897 { 2898 uint32_t data = 0; 2899 uint32_t default_data = 0; 2900 2901 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2902 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2903 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2904 enable ? 1 : 0); 2905 if(default_data != data) 2906 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2907 } 2908 2909 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2910 bool enable) 2911 { 2912 uint32_t data = 0; 2913 uint32_t default_data = 0; 2914 2915 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2916 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2917 CP_PG_DISABLE, 2918 enable ? 0 : 1); 2919 if(default_data != data) 2920 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2921 } 2922 2923 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2924 bool enable) 2925 { 2926 uint32_t data, default_data; 2927 2928 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2929 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2930 GFX_POWER_GATING_ENABLE, 2931 enable ? 1 : 0); 2932 if(default_data != data) 2933 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2934 } 2935 2936 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2937 bool enable) 2938 { 2939 uint32_t data, default_data; 2940 2941 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2942 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2943 GFX_PIPELINE_PG_ENABLE, 2944 enable ? 1 : 0); 2945 if(default_data != data) 2946 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2947 2948 if (!enable) 2949 /* read any GFX register to wake up GFX */ 2950 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2951 } 2952 2953 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2954 bool enable) 2955 { 2956 uint32_t data, default_data; 2957 2958 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2959 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2960 STATIC_PER_CU_PG_ENABLE, 2961 enable ? 1 : 0); 2962 if(default_data != data) 2963 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2964 } 2965 2966 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2967 bool enable) 2968 { 2969 uint32_t data, default_data; 2970 2971 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2972 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2973 DYN_PER_CU_PG_ENABLE, 2974 enable ? 1 : 0); 2975 if(default_data != data) 2976 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2977 } 2978 2979 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2980 { 2981 gfx_v9_0_init_csb(adev); 2982 2983 /* 2984 * Rlc save restore list is workable since v2_1. 2985 * And it's needed by gfxoff feature. 2986 */ 2987 if (adev->gfx.rlc.is_rlc_v2_1) { 2988 if (adev->asic_type == CHIP_VEGA12 || 2989 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 2990 gfx_v9_1_init_rlc_save_restore_list(adev); 2991 gfx_v9_0_enable_save_restore_machine(adev); 2992 } 2993 2994 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2995 AMD_PG_SUPPORT_GFX_SMG | 2996 AMD_PG_SUPPORT_GFX_DMG | 2997 AMD_PG_SUPPORT_CP | 2998 AMD_PG_SUPPORT_GDS | 2999 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3000 WREG32(mmRLC_JUMP_TABLE_RESTORE, 3001 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3002 gfx_v9_0_init_gfx_power_gating(adev); 3003 } 3004 } 3005 3006 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3007 { 3008 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3009 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3010 gfx_v9_0_wait_for_rlc_serdes(adev); 3011 } 3012 3013 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3014 { 3015 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3016 udelay(50); 3017 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3018 udelay(50); 3019 } 3020 3021 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3022 { 3023 #ifdef AMDGPU_RLC_DEBUG_RETRY 3024 u32 rlc_ucode_ver; 3025 #endif 3026 3027 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3028 udelay(50); 3029 3030 /* carrizo do enable cp interrupt after cp inited */ 3031 if (!(adev->flags & AMD_IS_APU)) { 3032 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3033 udelay(50); 3034 } 3035 3036 #ifdef AMDGPU_RLC_DEBUG_RETRY 3037 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3038 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3039 if(rlc_ucode_ver == 0x108) { 3040 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3041 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3042 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3043 * default is 0x9C4 to create a 100us interval */ 3044 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3045 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3046 * to disable the page fault retry interrupts, default is 3047 * 0x100 (256) */ 3048 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3049 } 3050 #endif 3051 } 3052 3053 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3054 { 3055 const struct rlc_firmware_header_v2_0 *hdr; 3056 const __le32 *fw_data; 3057 unsigned i, fw_size; 3058 3059 if (!adev->gfx.rlc_fw) 3060 return -EINVAL; 3061 3062 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3063 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3064 3065 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3066 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3067 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3068 3069 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3070 RLCG_UCODE_LOADING_START_ADDRESS); 3071 for (i = 0; i < fw_size; i++) 3072 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3073 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3074 3075 return 0; 3076 } 3077 3078 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3079 { 3080 int r; 3081 3082 if (amdgpu_sriov_vf(adev)) { 3083 gfx_v9_0_init_csb(adev); 3084 return 0; 3085 } 3086 3087 adev->gfx.rlc.funcs->stop(adev); 3088 3089 /* disable CG */ 3090 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3091 3092 gfx_v9_0_init_pg(adev); 3093 3094 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3095 /* legacy rlc firmware loading */ 3096 r = gfx_v9_0_rlc_load_microcode(adev); 3097 if (r) 3098 return r; 3099 } 3100 3101 switch (adev->asic_type) { 3102 case CHIP_RAVEN: 3103 if (amdgpu_lbpw == 0) 3104 gfx_v9_0_enable_lbpw(adev, false); 3105 else 3106 gfx_v9_0_enable_lbpw(adev, true); 3107 break; 3108 case CHIP_VEGA20: 3109 if (amdgpu_lbpw > 0) 3110 gfx_v9_0_enable_lbpw(adev, true); 3111 else 3112 gfx_v9_0_enable_lbpw(adev, false); 3113 break; 3114 default: 3115 break; 3116 } 3117 3118 adev->gfx.rlc.funcs->start(adev); 3119 3120 return 0; 3121 } 3122 3123 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3124 { 3125 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3126 3127 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3128 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3129 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3130 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3131 udelay(50); 3132 } 3133 3134 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3135 { 3136 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3137 const struct gfx_firmware_header_v1_0 *ce_hdr; 3138 const struct gfx_firmware_header_v1_0 *me_hdr; 3139 const __le32 *fw_data; 3140 unsigned i, fw_size; 3141 3142 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3143 return -EINVAL; 3144 3145 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3146 adev->gfx.pfp_fw->data; 3147 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3148 adev->gfx.ce_fw->data; 3149 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3150 adev->gfx.me_fw->data; 3151 3152 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3153 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3154 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3155 3156 gfx_v9_0_cp_gfx_enable(adev, false); 3157 3158 /* PFP */ 3159 fw_data = (const __le32 *) 3160 (adev->gfx.pfp_fw->data + 3161 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3162 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3163 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3164 for (i = 0; i < fw_size; i++) 3165 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3166 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3167 3168 /* CE */ 3169 fw_data = (const __le32 *) 3170 (adev->gfx.ce_fw->data + 3171 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3172 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3173 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3174 for (i = 0; i < fw_size; i++) 3175 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3176 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3177 3178 /* ME */ 3179 fw_data = (const __le32 *) 3180 (adev->gfx.me_fw->data + 3181 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3182 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3183 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3184 for (i = 0; i < fw_size; i++) 3185 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3186 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3187 3188 return 0; 3189 } 3190 3191 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3192 { 3193 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3194 const struct cs_section_def *sect = NULL; 3195 const struct cs_extent_def *ext = NULL; 3196 int r, i, tmp; 3197 3198 /* init the CP */ 3199 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3200 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3201 3202 gfx_v9_0_cp_gfx_enable(adev, true); 3203 3204 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3205 if (r) { 3206 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3207 return r; 3208 } 3209 3210 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3211 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3212 3213 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3214 amdgpu_ring_write(ring, 0x80000000); 3215 amdgpu_ring_write(ring, 0x80000000); 3216 3217 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3218 for (ext = sect->section; ext->extent != NULL; ++ext) { 3219 if (sect->id == SECT_CONTEXT) { 3220 amdgpu_ring_write(ring, 3221 PACKET3(PACKET3_SET_CONTEXT_REG, 3222 ext->reg_count)); 3223 amdgpu_ring_write(ring, 3224 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3225 for (i = 0; i < ext->reg_count; i++) 3226 amdgpu_ring_write(ring, ext->extent[i]); 3227 } 3228 } 3229 } 3230 3231 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3232 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3233 3234 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3235 amdgpu_ring_write(ring, 0); 3236 3237 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3238 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3239 amdgpu_ring_write(ring, 0x8000); 3240 amdgpu_ring_write(ring, 0x8000); 3241 3242 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3243 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3244 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3245 amdgpu_ring_write(ring, tmp); 3246 amdgpu_ring_write(ring, 0); 3247 3248 amdgpu_ring_commit(ring); 3249 3250 return 0; 3251 } 3252 3253 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3254 { 3255 struct amdgpu_ring *ring; 3256 u32 tmp; 3257 u32 rb_bufsz; 3258 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3259 3260 /* Set the write pointer delay */ 3261 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3262 3263 /* set the RB to use vmid 0 */ 3264 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3265 3266 /* Set ring buffer size */ 3267 ring = &adev->gfx.gfx_ring[0]; 3268 rb_bufsz = order_base_2(ring->ring_size / 8); 3269 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3270 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3271 #ifdef __BIG_ENDIAN 3272 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3273 #endif 3274 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3275 3276 /* Initialize the ring buffer's write pointers */ 3277 ring->wptr = 0; 3278 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3279 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3280 3281 /* set the wb address wether it's enabled or not */ 3282 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3283 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3284 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3285 3286 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3287 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3288 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3289 3290 mdelay(1); 3291 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3292 3293 rb_addr = ring->gpu_addr >> 8; 3294 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3295 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3296 3297 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3298 if (ring->use_doorbell) { 3299 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3300 DOORBELL_OFFSET, ring->doorbell_index); 3301 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3302 DOORBELL_EN, 1); 3303 } else { 3304 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3305 } 3306 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3307 3308 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3309 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3310 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3311 3312 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3313 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3314 3315 3316 /* start the ring */ 3317 gfx_v9_0_cp_gfx_start(adev); 3318 ring->sched.ready = true; 3319 3320 return 0; 3321 } 3322 3323 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3324 { 3325 if (enable) { 3326 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3327 } else { 3328 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3329 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3330 adev->gfx.kiq.ring.sched.ready = false; 3331 } 3332 udelay(50); 3333 } 3334 3335 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3336 { 3337 const struct gfx_firmware_header_v1_0 *mec_hdr; 3338 const __le32 *fw_data; 3339 unsigned i; 3340 u32 tmp; 3341 3342 if (!adev->gfx.mec_fw) 3343 return -EINVAL; 3344 3345 gfx_v9_0_cp_compute_enable(adev, false); 3346 3347 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3348 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3349 3350 fw_data = (const __le32 *) 3351 (adev->gfx.mec_fw->data + 3352 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3353 tmp = 0; 3354 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3355 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3356 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3357 3358 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3359 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3360 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3361 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3362 3363 /* MEC1 */ 3364 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3365 mec_hdr->jt_offset); 3366 for (i = 0; i < mec_hdr->jt_size; i++) 3367 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3368 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3369 3370 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3371 adev->gfx.mec_fw_version); 3372 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3373 3374 return 0; 3375 } 3376 3377 /* KIQ functions */ 3378 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3379 { 3380 uint32_t tmp; 3381 struct amdgpu_device *adev = ring->adev; 3382 3383 /* tell RLC which is KIQ queue */ 3384 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3385 tmp &= 0xffffff00; 3386 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3387 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3388 tmp |= 0x80; 3389 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3390 } 3391 3392 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3393 { 3394 struct amdgpu_device *adev = ring->adev; 3395 3396 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3397 if (amdgpu_gfx_is_high_priority_compute_queue(adev, 3398 ring->pipe, 3399 ring->queue)) { 3400 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3401 mqd->cp_hqd_queue_priority = 3402 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3403 } 3404 } 3405 } 3406 3407 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3408 { 3409 struct amdgpu_device *adev = ring->adev; 3410 struct v9_mqd *mqd = ring->mqd_ptr; 3411 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3412 uint32_t tmp; 3413 3414 mqd->header = 0xC0310800; 3415 mqd->compute_pipelinestat_enable = 0x00000001; 3416 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3417 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3418 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3419 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3420 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3421 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3422 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3423 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3424 mqd->compute_misc_reserved = 0x00000003; 3425 3426 mqd->dynamic_cu_mask_addr_lo = 3427 lower_32_bits(ring->mqd_gpu_addr 3428 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3429 mqd->dynamic_cu_mask_addr_hi = 3430 upper_32_bits(ring->mqd_gpu_addr 3431 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3432 3433 eop_base_addr = ring->eop_gpu_addr >> 8; 3434 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3435 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3436 3437 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3438 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3439 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3440 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3441 3442 mqd->cp_hqd_eop_control = tmp; 3443 3444 /* enable doorbell? */ 3445 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3446 3447 if (ring->use_doorbell) { 3448 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3449 DOORBELL_OFFSET, ring->doorbell_index); 3450 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3451 DOORBELL_EN, 1); 3452 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3453 DOORBELL_SOURCE, 0); 3454 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3455 DOORBELL_HIT, 0); 3456 } else { 3457 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3458 DOORBELL_EN, 0); 3459 } 3460 3461 mqd->cp_hqd_pq_doorbell_control = tmp; 3462 3463 /* disable the queue if it's active */ 3464 ring->wptr = 0; 3465 mqd->cp_hqd_dequeue_request = 0; 3466 mqd->cp_hqd_pq_rptr = 0; 3467 mqd->cp_hqd_pq_wptr_lo = 0; 3468 mqd->cp_hqd_pq_wptr_hi = 0; 3469 3470 /* set the pointer to the MQD */ 3471 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3472 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3473 3474 /* set MQD vmid to 0 */ 3475 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3476 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3477 mqd->cp_mqd_control = tmp; 3478 3479 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3480 hqd_gpu_addr = ring->gpu_addr >> 8; 3481 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3482 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3483 3484 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3485 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3486 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3487 (order_base_2(ring->ring_size / 4) - 1)); 3488 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3489 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3490 #ifdef __BIG_ENDIAN 3491 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3492 #endif 3493 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3494 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3495 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3496 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3497 mqd->cp_hqd_pq_control = tmp; 3498 3499 /* set the wb address whether it's enabled or not */ 3500 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3501 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3502 mqd->cp_hqd_pq_rptr_report_addr_hi = 3503 upper_32_bits(wb_gpu_addr) & 0xffff; 3504 3505 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3506 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3507 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3508 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3509 3510 tmp = 0; 3511 /* enable the doorbell if requested */ 3512 if (ring->use_doorbell) { 3513 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3514 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3515 DOORBELL_OFFSET, ring->doorbell_index); 3516 3517 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3518 DOORBELL_EN, 1); 3519 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3520 DOORBELL_SOURCE, 0); 3521 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3522 DOORBELL_HIT, 0); 3523 } 3524 3525 mqd->cp_hqd_pq_doorbell_control = tmp; 3526 3527 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3528 ring->wptr = 0; 3529 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3530 3531 /* set the vmid for the queue */ 3532 mqd->cp_hqd_vmid = 0; 3533 3534 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3535 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3536 mqd->cp_hqd_persistent_state = tmp; 3537 3538 /* set MIN_IB_AVAIL_SIZE */ 3539 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3540 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3541 mqd->cp_hqd_ib_control = tmp; 3542 3543 /* set static priority for a queue/ring */ 3544 gfx_v9_0_mqd_set_priority(ring, mqd); 3545 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); 3546 3547 /* map_queues packet doesn't need activate the queue, 3548 * so only kiq need set this field. 3549 */ 3550 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3551 mqd->cp_hqd_active = 1; 3552 3553 return 0; 3554 } 3555 3556 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3557 { 3558 struct amdgpu_device *adev = ring->adev; 3559 struct v9_mqd *mqd = ring->mqd_ptr; 3560 int j; 3561 3562 /* disable wptr polling */ 3563 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3564 3565 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3566 mqd->cp_hqd_eop_base_addr_lo); 3567 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3568 mqd->cp_hqd_eop_base_addr_hi); 3569 3570 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3571 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3572 mqd->cp_hqd_eop_control); 3573 3574 /* enable doorbell? */ 3575 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3576 mqd->cp_hqd_pq_doorbell_control); 3577 3578 /* disable the queue if it's active */ 3579 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3580 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3581 for (j = 0; j < adev->usec_timeout; j++) { 3582 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3583 break; 3584 udelay(1); 3585 } 3586 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3587 mqd->cp_hqd_dequeue_request); 3588 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3589 mqd->cp_hqd_pq_rptr); 3590 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3591 mqd->cp_hqd_pq_wptr_lo); 3592 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3593 mqd->cp_hqd_pq_wptr_hi); 3594 } 3595 3596 /* set the pointer to the MQD */ 3597 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3598 mqd->cp_mqd_base_addr_lo); 3599 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3600 mqd->cp_mqd_base_addr_hi); 3601 3602 /* set MQD vmid to 0 */ 3603 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3604 mqd->cp_mqd_control); 3605 3606 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3607 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3608 mqd->cp_hqd_pq_base_lo); 3609 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3610 mqd->cp_hqd_pq_base_hi); 3611 3612 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3613 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3614 mqd->cp_hqd_pq_control); 3615 3616 /* set the wb address whether it's enabled or not */ 3617 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3618 mqd->cp_hqd_pq_rptr_report_addr_lo); 3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3620 mqd->cp_hqd_pq_rptr_report_addr_hi); 3621 3622 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3623 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3624 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3625 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3626 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3627 3628 /* enable the doorbell if requested */ 3629 if (ring->use_doorbell) { 3630 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3631 (adev->doorbell_index.kiq * 2) << 2); 3632 /* If GC has entered CGPG, ringing doorbell > first page 3633 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to 3634 * workaround this issue. And this change has to align with firmware 3635 * update. 3636 */ 3637 if (check_if_enlarge_doorbell_range(adev)) 3638 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3639 (adev->doorbell.size - 4)); 3640 else 3641 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3642 (adev->doorbell_index.userqueue_end * 2) << 2); 3643 } 3644 3645 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3646 mqd->cp_hqd_pq_doorbell_control); 3647 3648 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3649 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3650 mqd->cp_hqd_pq_wptr_lo); 3651 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3652 mqd->cp_hqd_pq_wptr_hi); 3653 3654 /* set the vmid for the queue */ 3655 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3656 3657 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3658 mqd->cp_hqd_persistent_state); 3659 3660 /* activate the queue */ 3661 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3662 mqd->cp_hqd_active); 3663 3664 if (ring->use_doorbell) 3665 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3666 3667 return 0; 3668 } 3669 3670 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3671 { 3672 struct amdgpu_device *adev = ring->adev; 3673 int j; 3674 3675 /* disable the queue if it's active */ 3676 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3677 3678 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3679 3680 for (j = 0; j < adev->usec_timeout; j++) { 3681 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3682 break; 3683 udelay(1); 3684 } 3685 3686 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3687 DRM_DEBUG("KIQ dequeue request failed.\n"); 3688 3689 /* Manual disable if dequeue request times out */ 3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3691 } 3692 3693 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3694 0); 3695 } 3696 3697 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3698 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3699 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3700 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3701 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3702 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3703 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3704 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3705 3706 return 0; 3707 } 3708 3709 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3710 { 3711 struct amdgpu_device *adev = ring->adev; 3712 struct v9_mqd *mqd = ring->mqd_ptr; 3713 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3714 3715 gfx_v9_0_kiq_setting(ring); 3716 3717 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3718 /* reset MQD to a clean status */ 3719 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3720 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3721 3722 /* reset ring buffer */ 3723 ring->wptr = 0; 3724 amdgpu_ring_clear_ring(ring); 3725 3726 mutex_lock(&adev->srbm_mutex); 3727 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3728 gfx_v9_0_kiq_init_register(ring); 3729 soc15_grbm_select(adev, 0, 0, 0, 0); 3730 mutex_unlock(&adev->srbm_mutex); 3731 } else { 3732 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3733 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3734 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3735 mutex_lock(&adev->srbm_mutex); 3736 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3737 gfx_v9_0_mqd_init(ring); 3738 gfx_v9_0_kiq_init_register(ring); 3739 soc15_grbm_select(adev, 0, 0, 0, 0); 3740 mutex_unlock(&adev->srbm_mutex); 3741 3742 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3743 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3744 } 3745 3746 return 0; 3747 } 3748 3749 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3750 { 3751 struct amdgpu_device *adev = ring->adev; 3752 struct v9_mqd *mqd = ring->mqd_ptr; 3753 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3754 3755 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3756 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3757 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3758 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3759 mutex_lock(&adev->srbm_mutex); 3760 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3761 gfx_v9_0_mqd_init(ring); 3762 soc15_grbm_select(adev, 0, 0, 0, 0); 3763 mutex_unlock(&adev->srbm_mutex); 3764 3765 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3766 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3767 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3768 /* reset MQD to a clean status */ 3769 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3770 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3771 3772 /* reset ring buffer */ 3773 ring->wptr = 0; 3774 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3775 amdgpu_ring_clear_ring(ring); 3776 } else { 3777 amdgpu_ring_clear_ring(ring); 3778 } 3779 3780 return 0; 3781 } 3782 3783 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3784 { 3785 struct amdgpu_ring *ring; 3786 int r; 3787 3788 ring = &adev->gfx.kiq.ring; 3789 3790 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3791 if (unlikely(r != 0)) 3792 return r; 3793 3794 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3795 if (unlikely(r != 0)) 3796 return r; 3797 3798 gfx_v9_0_kiq_init_queue(ring); 3799 amdgpu_bo_kunmap(ring->mqd_obj); 3800 ring->mqd_ptr = NULL; 3801 amdgpu_bo_unreserve(ring->mqd_obj); 3802 ring->sched.ready = true; 3803 return 0; 3804 } 3805 3806 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3807 { 3808 struct amdgpu_ring *ring = NULL; 3809 int r = 0, i; 3810 3811 gfx_v9_0_cp_compute_enable(adev, true); 3812 3813 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3814 ring = &adev->gfx.compute_ring[i]; 3815 3816 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3817 if (unlikely(r != 0)) 3818 goto done; 3819 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3820 if (!r) { 3821 r = gfx_v9_0_kcq_init_queue(ring); 3822 amdgpu_bo_kunmap(ring->mqd_obj); 3823 ring->mqd_ptr = NULL; 3824 } 3825 amdgpu_bo_unreserve(ring->mqd_obj); 3826 if (r) 3827 goto done; 3828 } 3829 3830 r = amdgpu_gfx_enable_kcq(adev); 3831 done: 3832 return r; 3833 } 3834 3835 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3836 { 3837 int r, i; 3838 struct amdgpu_ring *ring; 3839 3840 if (!(adev->flags & AMD_IS_APU)) 3841 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3842 3843 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3844 if (adev->asic_type != CHIP_ARCTURUS) { 3845 /* legacy firmware loading */ 3846 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3847 if (r) 3848 return r; 3849 } 3850 3851 r = gfx_v9_0_cp_compute_load_microcode(adev); 3852 if (r) 3853 return r; 3854 } 3855 3856 r = gfx_v9_0_kiq_resume(adev); 3857 if (r) 3858 return r; 3859 3860 if (adev->asic_type != CHIP_ARCTURUS) { 3861 r = gfx_v9_0_cp_gfx_resume(adev); 3862 if (r) 3863 return r; 3864 } 3865 3866 r = gfx_v9_0_kcq_resume(adev); 3867 if (r) 3868 return r; 3869 3870 if (adev->asic_type != CHIP_ARCTURUS) { 3871 ring = &adev->gfx.gfx_ring[0]; 3872 r = amdgpu_ring_test_helper(ring); 3873 if (r) 3874 return r; 3875 } 3876 3877 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3878 ring = &adev->gfx.compute_ring[i]; 3879 amdgpu_ring_test_helper(ring); 3880 } 3881 3882 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3883 3884 return 0; 3885 } 3886 3887 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3888 { 3889 u32 tmp; 3890 3891 if (adev->asic_type != CHIP_ARCTURUS) 3892 return; 3893 3894 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3895 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3896 adev->df.hash_status.hash_64k); 3897 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3898 adev->df.hash_status.hash_2m); 3899 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3900 adev->df.hash_status.hash_1g); 3901 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3902 } 3903 3904 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3905 { 3906 if (adev->asic_type != CHIP_ARCTURUS) 3907 gfx_v9_0_cp_gfx_enable(adev, enable); 3908 gfx_v9_0_cp_compute_enable(adev, enable); 3909 } 3910 3911 static int gfx_v9_0_hw_init(void *handle) 3912 { 3913 int r; 3914 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3915 3916 if (!amdgpu_sriov_vf(adev)) 3917 gfx_v9_0_init_golden_registers(adev); 3918 3919 gfx_v9_0_constants_init(adev); 3920 3921 gfx_v9_0_init_tcp_config(adev); 3922 3923 r = adev->gfx.rlc.funcs->resume(adev); 3924 if (r) 3925 return r; 3926 3927 r = gfx_v9_0_cp_resume(adev); 3928 if (r) 3929 return r; 3930 3931 return r; 3932 } 3933 3934 static int gfx_v9_0_hw_fini(void *handle) 3935 { 3936 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3937 3938 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3939 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3940 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3941 3942 /* DF freeze and kcq disable will fail */ 3943 if (!amdgpu_ras_intr_triggered()) 3944 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3945 amdgpu_gfx_disable_kcq(adev); 3946 3947 if (amdgpu_sriov_vf(adev)) { 3948 gfx_v9_0_cp_gfx_enable(adev, false); 3949 /* must disable polling for SRIOV when hw finished, otherwise 3950 * CPC engine may still keep fetching WB address which is already 3951 * invalid after sw finished and trigger DMAR reading error in 3952 * hypervisor side. 3953 */ 3954 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3955 return 0; 3956 } 3957 3958 /* Use deinitialize sequence from CAIL when unbinding device from driver, 3959 * otherwise KIQ is hanging when binding back 3960 */ 3961 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3962 mutex_lock(&adev->srbm_mutex); 3963 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 3964 adev->gfx.kiq.ring.pipe, 3965 adev->gfx.kiq.ring.queue, 0); 3966 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 3967 soc15_grbm_select(adev, 0, 0, 0, 0); 3968 mutex_unlock(&adev->srbm_mutex); 3969 } 3970 3971 gfx_v9_0_cp_enable(adev, false); 3972 adev->gfx.rlc.funcs->stop(adev); 3973 3974 return 0; 3975 } 3976 3977 static int gfx_v9_0_suspend(void *handle) 3978 { 3979 return gfx_v9_0_hw_fini(handle); 3980 } 3981 3982 static int gfx_v9_0_resume(void *handle) 3983 { 3984 return gfx_v9_0_hw_init(handle); 3985 } 3986 3987 static bool gfx_v9_0_is_idle(void *handle) 3988 { 3989 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3990 3991 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3992 GRBM_STATUS, GUI_ACTIVE)) 3993 return false; 3994 else 3995 return true; 3996 } 3997 3998 static int gfx_v9_0_wait_for_idle(void *handle) 3999 { 4000 unsigned i; 4001 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4002 4003 for (i = 0; i < adev->usec_timeout; i++) { 4004 if (gfx_v9_0_is_idle(handle)) 4005 return 0; 4006 udelay(1); 4007 } 4008 return -ETIMEDOUT; 4009 } 4010 4011 static int gfx_v9_0_soft_reset(void *handle) 4012 { 4013 u32 grbm_soft_reset = 0; 4014 u32 tmp; 4015 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4016 4017 /* GRBM_STATUS */ 4018 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4019 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4020 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4021 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4022 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4023 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4024 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4025 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4026 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4027 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4028 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4029 } 4030 4031 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4032 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4033 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4034 } 4035 4036 /* GRBM_STATUS2 */ 4037 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4038 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4039 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4040 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4041 4042 4043 if (grbm_soft_reset) { 4044 /* stop the rlc */ 4045 adev->gfx.rlc.funcs->stop(adev); 4046 4047 if (adev->asic_type != CHIP_ARCTURUS) 4048 /* Disable GFX parsing/prefetching */ 4049 gfx_v9_0_cp_gfx_enable(adev, false); 4050 4051 /* Disable MEC parsing/prefetching */ 4052 gfx_v9_0_cp_compute_enable(adev, false); 4053 4054 if (grbm_soft_reset) { 4055 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4056 tmp |= grbm_soft_reset; 4057 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4058 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4059 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4060 4061 udelay(50); 4062 4063 tmp &= ~grbm_soft_reset; 4064 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4065 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4066 } 4067 4068 /* Wait a little for things to settle down */ 4069 udelay(50); 4070 } 4071 return 0; 4072 } 4073 4074 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4075 { 4076 signed long r, cnt = 0; 4077 unsigned long flags; 4078 uint32_t seq, reg_val_offs = 0; 4079 uint64_t value = 0; 4080 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4081 struct amdgpu_ring *ring = &kiq->ring; 4082 4083 BUG_ON(!ring->funcs->emit_rreg); 4084 4085 spin_lock_irqsave(&kiq->ring_lock, flags); 4086 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4087 pr_err("critical bug! too many kiq readers\n"); 4088 goto failed_unlock; 4089 } 4090 amdgpu_ring_alloc(ring, 32); 4091 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4092 amdgpu_ring_write(ring, 9 | /* src: register*/ 4093 (5 << 8) | /* dst: memory */ 4094 (1 << 16) | /* count sel */ 4095 (1 << 20)); /* write confirm */ 4096 amdgpu_ring_write(ring, 0); 4097 amdgpu_ring_write(ring, 0); 4098 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4099 reg_val_offs * 4)); 4100 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4101 reg_val_offs * 4)); 4102 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4103 if (r) 4104 goto failed_undo; 4105 4106 amdgpu_ring_commit(ring); 4107 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4108 4109 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4110 4111 /* don't wait anymore for gpu reset case because this way may 4112 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4113 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4114 * never return if we keep waiting in virt_kiq_rreg, which cause 4115 * gpu_recover() hang there. 4116 * 4117 * also don't wait anymore for IRQ context 4118 * */ 4119 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 4120 goto failed_kiq_read; 4121 4122 might_sleep(); 4123 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4124 drm_msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4125 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4126 } 4127 4128 if (cnt > MAX_KIQ_REG_TRY) 4129 goto failed_kiq_read; 4130 4131 mb(); 4132 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4133 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4134 amdgpu_device_wb_free(adev, reg_val_offs); 4135 return value; 4136 4137 failed_undo: 4138 amdgpu_ring_undo(ring); 4139 failed_unlock: 4140 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4141 failed_kiq_read: 4142 if (reg_val_offs) 4143 amdgpu_device_wb_free(adev, reg_val_offs); 4144 pr_err("failed to read gpu clock\n"); 4145 return ~0; 4146 } 4147 4148 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4149 { 4150 uint64_t clock; 4151 4152 amdgpu_gfx_off_ctrl(adev, false); 4153 mutex_lock(&adev->gfx.gpu_clock_mutex); 4154 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4155 clock = gfx_v9_0_kiq_read_clock(adev); 4156 } else { 4157 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4158 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4159 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4160 } 4161 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4162 amdgpu_gfx_off_ctrl(adev, true); 4163 return clock; 4164 } 4165 4166 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4167 uint32_t vmid, 4168 uint32_t gds_base, uint32_t gds_size, 4169 uint32_t gws_base, uint32_t gws_size, 4170 uint32_t oa_base, uint32_t oa_size) 4171 { 4172 struct amdgpu_device *adev = ring->adev; 4173 4174 /* GDS Base */ 4175 gfx_v9_0_write_data_to_reg(ring, 0, false, 4176 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4177 gds_base); 4178 4179 /* GDS Size */ 4180 gfx_v9_0_write_data_to_reg(ring, 0, false, 4181 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4182 gds_size); 4183 4184 /* GWS */ 4185 gfx_v9_0_write_data_to_reg(ring, 0, false, 4186 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4187 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4188 4189 /* OA */ 4190 gfx_v9_0_write_data_to_reg(ring, 0, false, 4191 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4192 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4193 } 4194 4195 static const u32 vgpr_init_compute_shader[] = 4196 { 4197 0xb07c0000, 0xbe8000ff, 4198 0x000000f8, 0xbf110800, 4199 0x7e000280, 0x7e020280, 4200 0x7e040280, 0x7e060280, 4201 0x7e080280, 0x7e0a0280, 4202 0x7e0c0280, 0x7e0e0280, 4203 0x80808800, 0xbe803200, 4204 0xbf84fff5, 0xbf9c0000, 4205 0xd28c0001, 0x0001007f, 4206 0xd28d0001, 0x0002027e, 4207 0x10020288, 0xb8810904, 4208 0xb7814000, 0xd1196a01, 4209 0x00000301, 0xbe800087, 4210 0xbefc00c1, 0xd89c4000, 4211 0x00020201, 0xd89cc080, 4212 0x00040401, 0x320202ff, 4213 0x00000800, 0x80808100, 4214 0xbf84fff8, 0x7e020280, 4215 0xbf810000, 0x00000000, 4216 }; 4217 4218 static const u32 sgpr_init_compute_shader[] = 4219 { 4220 0xb07c0000, 0xbe8000ff, 4221 0x0000005f, 0xbee50080, 4222 0xbe812c65, 0xbe822c65, 4223 0xbe832c65, 0xbe842c65, 4224 0xbe852c65, 0xb77c0005, 4225 0x80808500, 0xbf84fff8, 4226 0xbe800080, 0xbf810000, 4227 }; 4228 4229 static const u32 vgpr_init_compute_shader_arcturus[] = { 4230 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4231 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4232 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4233 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4234 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4235 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4236 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4237 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4238 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4239 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4240 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4241 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4242 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4243 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4244 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4245 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4246 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4247 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4248 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4249 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4250 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4251 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4252 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4253 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4254 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4255 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4256 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4257 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4258 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4259 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4260 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4261 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4262 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4263 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4264 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4265 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4266 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4267 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4268 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4269 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4270 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4271 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4272 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4273 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4274 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4275 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4276 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4277 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4278 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4279 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4280 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4281 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4282 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4283 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4284 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4285 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4286 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4287 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4288 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4289 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4290 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4291 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4292 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4293 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4294 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4295 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4296 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4297 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4298 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4299 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4300 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4301 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4302 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4303 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4304 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4305 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4306 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4307 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4308 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4309 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4310 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4311 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4312 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4313 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4314 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4315 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4316 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4317 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4318 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4319 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4320 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4321 0xbf84fff8, 0xbf810000, 4322 }; 4323 4324 /* When below register arrays changed, please update gpr_reg_size, 4325 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4326 to cover all gfx9 ASICs */ 4327 static const struct soc15_reg_entry vgpr_init_regs[] = { 4328 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4329 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4330 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4331 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4332 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4333 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4334 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4335 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4336 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4337 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4338 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4339 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4340 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4341 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4342 }; 4343 4344 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4345 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4346 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4347 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4348 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4349 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4350 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4351 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4352 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4353 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4354 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4355 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4356 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4357 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4358 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4359 }; 4360 4361 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4362 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4363 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4364 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4365 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4366 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4367 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4368 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4369 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4370 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4371 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4372 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4373 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4374 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4375 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4376 }; 4377 4378 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4379 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4380 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4381 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4382 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4383 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4384 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4385 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4386 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4387 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4388 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4389 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4390 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4391 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4392 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4393 }; 4394 4395 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4396 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4397 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4398 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4399 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4400 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4401 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4402 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4403 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4404 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4405 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4406 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4407 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4408 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4409 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4410 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4411 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4412 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4413 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4414 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4415 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4416 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4417 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4418 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4419 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4420 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4421 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4422 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4423 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4424 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4425 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4426 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4427 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4428 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4429 }; 4430 4431 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4432 { 4433 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4434 int i, r; 4435 4436 /* only support when RAS is enabled */ 4437 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4438 return 0; 4439 4440 r = amdgpu_ring_alloc(ring, 7); 4441 if (r) { 4442 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4443 ring->name, r); 4444 return r; 4445 } 4446 4447 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4448 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4449 4450 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4451 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4452 PACKET3_DMA_DATA_DST_SEL(1) | 4453 PACKET3_DMA_DATA_SRC_SEL(2) | 4454 PACKET3_DMA_DATA_ENGINE(0))); 4455 amdgpu_ring_write(ring, 0); 4456 amdgpu_ring_write(ring, 0); 4457 amdgpu_ring_write(ring, 0); 4458 amdgpu_ring_write(ring, 0); 4459 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4460 adev->gds.gds_size); 4461 4462 amdgpu_ring_commit(ring); 4463 4464 for (i = 0; i < adev->usec_timeout; i++) { 4465 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4466 break; 4467 udelay(1); 4468 } 4469 4470 if (i >= adev->usec_timeout) 4471 r = -ETIMEDOUT; 4472 4473 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4474 4475 return r; 4476 } 4477 4478 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4479 { 4480 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4481 struct amdgpu_ib ib; 4482 struct dma_fence *f = NULL; 4483 int r, i; 4484 unsigned total_size, vgpr_offset, sgpr_offset; 4485 u64 gpu_addr; 4486 4487 int compute_dim_x = adev->gfx.config.max_shader_engines * 4488 adev->gfx.config.max_cu_per_sh * 4489 adev->gfx.config.max_sh_per_se; 4490 int sgpr_work_group_size = 5; 4491 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4492 int vgpr_init_shader_size; 4493 const u32 *vgpr_init_shader_ptr; 4494 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4495 4496 /* only support when RAS is enabled */ 4497 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4498 return 0; 4499 4500 /* bail if the compute ring is not ready */ 4501 if (!ring->sched.ready) 4502 return 0; 4503 4504 if (adev->asic_type == CHIP_ARCTURUS) { 4505 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4506 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4507 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4508 } else { 4509 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4510 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4511 vgpr_init_regs_ptr = vgpr_init_regs; 4512 } 4513 4514 total_size = 4515 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4516 total_size += 4517 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4518 total_size += 4519 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4520 total_size = roundup2(total_size, 256); 4521 vgpr_offset = total_size; 4522 total_size += roundup2(vgpr_init_shader_size, 256); 4523 sgpr_offset = total_size; 4524 total_size += sizeof(sgpr_init_compute_shader); 4525 4526 /* allocate an indirect buffer to put the commands in */ 4527 memset(&ib, 0, sizeof(ib)); 4528 r = amdgpu_ib_get(adev, NULL, total_size, 4529 AMDGPU_IB_POOL_DIRECT, &ib); 4530 if (r) { 4531 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4532 return r; 4533 } 4534 4535 /* load the compute shaders */ 4536 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4537 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4538 4539 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4540 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4541 4542 /* init the ib length to 0 */ 4543 ib.length_dw = 0; 4544 4545 /* VGPR */ 4546 /* write the register state for the compute dispatch */ 4547 for (i = 0; i < gpr_reg_size; i++) { 4548 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4549 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4550 - PACKET3_SET_SH_REG_START; 4551 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4552 } 4553 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4554 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4555 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4556 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4557 - PACKET3_SET_SH_REG_START; 4558 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4559 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4560 4561 /* write dispatch packet */ 4562 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4563 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4564 ib.ptr[ib.length_dw++] = 1; /* y */ 4565 ib.ptr[ib.length_dw++] = 1; /* z */ 4566 ib.ptr[ib.length_dw++] = 4567 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4568 4569 /* write CS partial flush packet */ 4570 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4571 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4572 4573 /* SGPR1 */ 4574 /* write the register state for the compute dispatch */ 4575 for (i = 0; i < gpr_reg_size; i++) { 4576 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4577 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4578 - PACKET3_SET_SH_REG_START; 4579 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4580 } 4581 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4582 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4583 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4584 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4585 - PACKET3_SET_SH_REG_START; 4586 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4587 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4588 4589 /* write dispatch packet */ 4590 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4591 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4592 ib.ptr[ib.length_dw++] = 1; /* y */ 4593 ib.ptr[ib.length_dw++] = 1; /* z */ 4594 ib.ptr[ib.length_dw++] = 4595 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4596 4597 /* write CS partial flush packet */ 4598 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4599 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4600 4601 /* SGPR2 */ 4602 /* write the register state for the compute dispatch */ 4603 for (i = 0; i < gpr_reg_size; i++) { 4604 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4605 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4606 - PACKET3_SET_SH_REG_START; 4607 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4608 } 4609 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4610 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4611 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4612 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4613 - PACKET3_SET_SH_REG_START; 4614 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4615 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4616 4617 /* write dispatch packet */ 4618 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4619 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4620 ib.ptr[ib.length_dw++] = 1; /* y */ 4621 ib.ptr[ib.length_dw++] = 1; /* z */ 4622 ib.ptr[ib.length_dw++] = 4623 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4624 4625 /* write CS partial flush packet */ 4626 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4627 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4628 4629 /* shedule the ib on the ring */ 4630 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4631 if (r) { 4632 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4633 goto fail; 4634 } 4635 4636 /* wait for the GPU to finish processing the IB */ 4637 r = dma_fence_wait(f, false); 4638 if (r) { 4639 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4640 goto fail; 4641 } 4642 4643 fail: 4644 amdgpu_ib_free(adev, &ib, NULL); 4645 dma_fence_put(f); 4646 4647 return r; 4648 } 4649 4650 static int gfx_v9_0_early_init(void *handle) 4651 { 4652 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4653 4654 if (adev->asic_type == CHIP_ARCTURUS) 4655 adev->gfx.num_gfx_rings = 0; 4656 else 4657 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4658 adev->gfx.num_compute_rings = amdgpu_num_kcq; 4659 gfx_v9_0_set_kiq_pm4_funcs(adev); 4660 gfx_v9_0_set_ring_funcs(adev); 4661 gfx_v9_0_set_irq_funcs(adev); 4662 gfx_v9_0_set_gds_init(adev); 4663 gfx_v9_0_set_rlc_funcs(adev); 4664 4665 return 0; 4666 } 4667 4668 static int gfx_v9_0_ecc_late_init(void *handle) 4669 { 4670 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4671 int r; 4672 4673 /* 4674 * Temp workaround to fix the issue that CP firmware fails to 4675 * update read pointer when CPDMA is writing clearing operation 4676 * to GDS in suspend/resume sequence on several cards. So just 4677 * limit this operation in cold boot sequence. 4678 */ 4679 if (!adev->in_suspend) { 4680 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4681 if (r) 4682 return r; 4683 } 4684 4685 /* requires IBs so do in late init after IB pool is initialized */ 4686 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4687 if (r) 4688 return r; 4689 4690 if (adev->gfx.funcs && 4691 adev->gfx.funcs->reset_ras_error_count) 4692 adev->gfx.funcs->reset_ras_error_count(adev); 4693 4694 r = amdgpu_gfx_ras_late_init(adev); 4695 if (r) 4696 return r; 4697 4698 return 0; 4699 } 4700 4701 static int gfx_v9_0_late_init(void *handle) 4702 { 4703 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4704 int r; 4705 4706 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4707 if (r) 4708 return r; 4709 4710 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4711 if (r) 4712 return r; 4713 4714 r = gfx_v9_0_ecc_late_init(handle); 4715 if (r) 4716 return r; 4717 4718 return 0; 4719 } 4720 4721 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4722 { 4723 uint32_t rlc_setting; 4724 4725 /* if RLC is not enabled, do nothing */ 4726 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4727 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4728 return false; 4729 4730 return true; 4731 } 4732 4733 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4734 { 4735 uint32_t data; 4736 unsigned i; 4737 4738 data = RLC_SAFE_MODE__CMD_MASK; 4739 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4740 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4741 4742 /* wait for RLC_SAFE_MODE */ 4743 for (i = 0; i < adev->usec_timeout; i++) { 4744 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4745 break; 4746 udelay(1); 4747 } 4748 } 4749 4750 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4751 { 4752 uint32_t data; 4753 4754 data = RLC_SAFE_MODE__CMD_MASK; 4755 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4756 } 4757 4758 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4759 bool enable) 4760 { 4761 amdgpu_gfx_rlc_enter_safe_mode(adev); 4762 4763 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4764 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4765 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4766 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4767 } else { 4768 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4769 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4770 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4771 } 4772 4773 amdgpu_gfx_rlc_exit_safe_mode(adev); 4774 } 4775 4776 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4777 bool enable) 4778 { 4779 /* TODO: double check if we need to perform under safe mode */ 4780 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4781 4782 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4783 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4784 else 4785 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4786 4787 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4788 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4789 else 4790 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4791 4792 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4793 } 4794 4795 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4796 bool enable) 4797 { 4798 uint32_t data, def; 4799 4800 amdgpu_gfx_rlc_enter_safe_mode(adev); 4801 4802 /* It is disabled by HW by default */ 4803 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4804 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4805 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4806 4807 if (adev->asic_type != CHIP_VEGA12) 4808 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4809 4810 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4811 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4812 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4813 4814 /* only for Vega10 & Raven1 */ 4815 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4816 4817 if (def != data) 4818 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4819 4820 /* MGLS is a global flag to control all MGLS in GFX */ 4821 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4822 /* 2 - RLC memory Light sleep */ 4823 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4824 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4825 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4826 if (def != data) 4827 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4828 } 4829 /* 3 - CP memory Light sleep */ 4830 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4831 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4832 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4833 if (def != data) 4834 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4835 } 4836 } 4837 } else { 4838 /* 1 - MGCG_OVERRIDE */ 4839 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4840 4841 if (adev->asic_type != CHIP_VEGA12) 4842 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4843 4844 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4845 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4846 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4847 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4848 4849 if (def != data) 4850 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4851 4852 /* 2 - disable MGLS in RLC */ 4853 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4854 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4855 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4856 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4857 } 4858 4859 /* 3 - disable MGLS in CP */ 4860 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4861 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4862 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4863 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4864 } 4865 } 4866 4867 amdgpu_gfx_rlc_exit_safe_mode(adev); 4868 } 4869 4870 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4871 bool enable) 4872 { 4873 uint32_t data, def; 4874 4875 if (adev->asic_type == CHIP_ARCTURUS) 4876 return; 4877 4878 amdgpu_gfx_rlc_enter_safe_mode(adev); 4879 4880 /* Enable 3D CGCG/CGLS */ 4881 if (enable) { 4882 /* write cmd to clear cgcg/cgls ov */ 4883 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4884 /* unset CGCG override */ 4885 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4886 /* update CGCG and CGLS override bits */ 4887 if (def != data) 4888 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4889 4890 /* enable 3Dcgcg FSM(0x0000363f) */ 4891 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4892 4893 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 4894 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4895 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4896 else 4897 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 4898 4899 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4900 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4901 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4902 if (def != data) 4903 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4904 4905 /* set IDLE_POLL_COUNT(0x00900100) */ 4906 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4907 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4908 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4909 if (def != data) 4910 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4911 } else { 4912 /* Disable CGCG/CGLS */ 4913 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4914 /* disable cgcg, cgls should be disabled */ 4915 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4916 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4917 /* disable cgcg and cgls in FSM */ 4918 if (def != data) 4919 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4920 } 4921 4922 amdgpu_gfx_rlc_exit_safe_mode(adev); 4923 } 4924 4925 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4926 bool enable) 4927 { 4928 uint32_t def, data; 4929 4930 amdgpu_gfx_rlc_enter_safe_mode(adev); 4931 4932 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4933 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4934 /* unset CGCG override */ 4935 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4936 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4937 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4938 else 4939 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4940 /* update CGCG and CGLS override bits */ 4941 if (def != data) 4942 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4943 4944 /* enable cgcg FSM(0x0000363F) */ 4945 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4946 4947 if (adev->asic_type == CHIP_ARCTURUS) 4948 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4949 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4950 else 4951 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4952 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4953 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4954 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4955 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4956 if (def != data) 4957 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4958 4959 /* set IDLE_POLL_COUNT(0x00900100) */ 4960 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4961 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4962 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4963 if (def != data) 4964 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4965 } else { 4966 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4967 /* reset CGCG/CGLS bits */ 4968 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4969 /* disable cgcg and cgls in FSM */ 4970 if (def != data) 4971 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4972 } 4973 4974 amdgpu_gfx_rlc_exit_safe_mode(adev); 4975 } 4976 4977 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4978 bool enable) 4979 { 4980 if (enable) { 4981 /* CGCG/CGLS should be enabled after MGCG/MGLS 4982 * === MGCG + MGLS === 4983 */ 4984 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4985 /* === CGCG /CGLS for GFX 3D Only === */ 4986 gfx_v9_0_update_3d_clock_gating(adev, enable); 4987 /* === CGCG + CGLS === */ 4988 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4989 } else { 4990 /* CGCG/CGLS should be disabled before MGCG/MGLS 4991 * === CGCG + CGLS === 4992 */ 4993 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4994 /* === CGCG /CGLS for GFX 3D Only === */ 4995 gfx_v9_0_update_3d_clock_gating(adev, enable); 4996 /* === MGCG + MGLS === */ 4997 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4998 } 4999 return 0; 5000 } 5001 5002 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5003 { 5004 u32 reg, data; 5005 5006 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5007 if (amdgpu_sriov_is_pp_one_vf(adev)) 5008 data = RREG32_NO_KIQ(reg); 5009 else 5010 data = RREG32(reg); 5011 5012 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5013 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5014 5015 if (amdgpu_sriov_is_pp_one_vf(adev)) 5016 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5017 else 5018 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5019 } 5020 5021 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5022 uint32_t offset, 5023 struct soc15_reg_rlcg *entries, int arr_size) 5024 { 5025 int i; 5026 uint32_t reg; 5027 5028 if (!entries) 5029 return false; 5030 5031 for (i = 0; i < arr_size; i++) { 5032 const struct soc15_reg_rlcg *entry; 5033 5034 entry = &entries[i]; 5035 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5036 if (offset == reg) 5037 return true; 5038 } 5039 5040 return false; 5041 } 5042 5043 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5044 { 5045 return gfx_v9_0_check_rlcg_range(adev, offset, 5046 (void *)rlcg_access_gc_9_0, 5047 ARRAY_SIZE(rlcg_access_gc_9_0)); 5048 } 5049 5050 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5051 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5052 .set_safe_mode = gfx_v9_0_set_safe_mode, 5053 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5054 .init = gfx_v9_0_rlc_init, 5055 .get_csb_size = gfx_v9_0_get_csb_size, 5056 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5057 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5058 .resume = gfx_v9_0_rlc_resume, 5059 .stop = gfx_v9_0_rlc_stop, 5060 .reset = gfx_v9_0_rlc_reset, 5061 .start = gfx_v9_0_rlc_start, 5062 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5063 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5064 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5065 }; 5066 5067 static int gfx_v9_0_set_powergating_state(void *handle, 5068 enum amd_powergating_state state) 5069 { 5070 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5071 bool enable = (state == AMD_PG_STATE_GATE); 5072 5073 switch (adev->asic_type) { 5074 case CHIP_RAVEN: 5075 case CHIP_RENOIR: 5076 if (!enable) 5077 amdgpu_gfx_off_ctrl(adev, false); 5078 5079 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5080 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5081 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5082 } else { 5083 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5084 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5085 } 5086 5087 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5088 gfx_v9_0_enable_cp_power_gating(adev, true); 5089 else 5090 gfx_v9_0_enable_cp_power_gating(adev, false); 5091 5092 /* update gfx cgpg state */ 5093 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5094 5095 /* update mgcg state */ 5096 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5097 5098 if (enable) 5099 amdgpu_gfx_off_ctrl(adev, true); 5100 break; 5101 case CHIP_VEGA12: 5102 amdgpu_gfx_off_ctrl(adev, enable); 5103 break; 5104 default: 5105 break; 5106 } 5107 5108 return 0; 5109 } 5110 5111 static int gfx_v9_0_set_clockgating_state(void *handle, 5112 enum amd_clockgating_state state) 5113 { 5114 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5115 5116 if (amdgpu_sriov_vf(adev)) 5117 return 0; 5118 5119 switch (adev->asic_type) { 5120 case CHIP_VEGA10: 5121 case CHIP_VEGA12: 5122 case CHIP_VEGA20: 5123 case CHIP_RAVEN: 5124 case CHIP_ARCTURUS: 5125 case CHIP_RENOIR: 5126 gfx_v9_0_update_gfx_clock_gating(adev, 5127 state == AMD_CG_STATE_GATE); 5128 break; 5129 default: 5130 break; 5131 } 5132 return 0; 5133 } 5134 5135 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5136 { 5137 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5138 int data; 5139 5140 if (amdgpu_sriov_vf(adev)) 5141 *flags = 0; 5142 5143 /* AMD_CG_SUPPORT_GFX_MGCG */ 5144 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5145 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5146 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5147 5148 /* AMD_CG_SUPPORT_GFX_CGCG */ 5149 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5150 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5151 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5152 5153 /* AMD_CG_SUPPORT_GFX_CGLS */ 5154 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5155 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5156 5157 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5158 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5159 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5160 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5161 5162 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5163 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5164 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5165 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5166 5167 if (adev->asic_type != CHIP_ARCTURUS) { 5168 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5169 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5170 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5171 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5172 5173 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5174 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5175 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5176 } 5177 } 5178 5179 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5180 { 5181 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5182 } 5183 5184 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5185 { 5186 struct amdgpu_device *adev = ring->adev; 5187 u64 wptr; 5188 5189 /* XXX check if swapping is necessary on BE */ 5190 if (ring->use_doorbell) { 5191 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5192 } else { 5193 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5194 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5195 } 5196 5197 return wptr; 5198 } 5199 5200 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5201 { 5202 struct amdgpu_device *adev = ring->adev; 5203 5204 if (ring->use_doorbell) { 5205 /* XXX check if swapping is necessary on BE */ 5206 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5207 WDOORBELL64(ring->doorbell_index, ring->wptr); 5208 } else { 5209 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5210 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5211 } 5212 } 5213 5214 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5215 { 5216 struct amdgpu_device *adev = ring->adev; 5217 u32 ref_and_mask, reg_mem_engine; 5218 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5219 5220 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5221 switch (ring->me) { 5222 case 1: 5223 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5224 break; 5225 case 2: 5226 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5227 break; 5228 default: 5229 return; 5230 } 5231 reg_mem_engine = 0; 5232 } else { 5233 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5234 reg_mem_engine = 1; /* pfp */ 5235 } 5236 5237 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5238 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5239 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5240 ref_and_mask, ref_and_mask, 0x20); 5241 } 5242 5243 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5244 struct amdgpu_job *job, 5245 struct amdgpu_ib *ib, 5246 uint32_t flags) 5247 { 5248 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5249 u32 header, control = 0; 5250 5251 if (ib->flags & AMDGPU_IB_FLAG_CE) 5252 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5253 else 5254 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5255 5256 control |= ib->length_dw | (vmid << 24); 5257 5258 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5259 control |= INDIRECT_BUFFER_PRE_ENB(1); 5260 5261 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5262 gfx_v9_0_ring_emit_de_meta(ring); 5263 } 5264 5265 amdgpu_ring_write(ring, header); 5266 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5267 amdgpu_ring_write(ring, 5268 #ifdef __BIG_ENDIAN 5269 (2 << 0) | 5270 #endif 5271 lower_32_bits(ib->gpu_addr)); 5272 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5273 amdgpu_ring_write(ring, control); 5274 } 5275 5276 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5277 struct amdgpu_job *job, 5278 struct amdgpu_ib *ib, 5279 uint32_t flags) 5280 { 5281 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5282 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5283 5284 /* Currently, there is a high possibility to get wave ID mismatch 5285 * between ME and GDS, leading to a hw deadlock, because ME generates 5286 * different wave IDs than the GDS expects. This situation happens 5287 * randomly when at least 5 compute pipes use GDS ordered append. 5288 * The wave IDs generated by ME are also wrong after suspend/resume. 5289 * Those are probably bugs somewhere else in the kernel driver. 5290 * 5291 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5292 * GDS to 0 for this ring (me/pipe). 5293 */ 5294 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5295 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5296 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5297 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5298 } 5299 5300 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5301 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5302 amdgpu_ring_write(ring, 5303 #ifdef __BIG_ENDIAN 5304 (2 << 0) | 5305 #endif 5306 lower_32_bits(ib->gpu_addr)); 5307 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5308 amdgpu_ring_write(ring, control); 5309 } 5310 5311 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5312 u64 seq, unsigned flags) 5313 { 5314 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5315 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5316 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5317 5318 /* RELEASE_MEM - flush caches, send int */ 5319 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5320 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5321 EOP_TC_NC_ACTION_EN) : 5322 (EOP_TCL1_ACTION_EN | 5323 EOP_TC_ACTION_EN | 5324 EOP_TC_WB_ACTION_EN | 5325 EOP_TC_MD_ACTION_EN)) | 5326 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5327 EVENT_INDEX(5))); 5328 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5329 5330 /* 5331 * the address should be Qword aligned if 64bit write, Dword 5332 * aligned if only send 32bit data low (discard data high) 5333 */ 5334 if (write64bit) 5335 BUG_ON(addr & 0x7); 5336 else 5337 BUG_ON(addr & 0x3); 5338 amdgpu_ring_write(ring, lower_32_bits(addr)); 5339 amdgpu_ring_write(ring, upper_32_bits(addr)); 5340 amdgpu_ring_write(ring, lower_32_bits(seq)); 5341 amdgpu_ring_write(ring, upper_32_bits(seq)); 5342 amdgpu_ring_write(ring, 0); 5343 } 5344 5345 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5346 { 5347 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5348 uint32_t seq = ring->fence_drv.sync_seq; 5349 uint64_t addr = ring->fence_drv.gpu_addr; 5350 5351 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5352 lower_32_bits(addr), upper_32_bits(addr), 5353 seq, 0xffffffff, 4); 5354 } 5355 5356 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5357 unsigned vmid, uint64_t pd_addr) 5358 { 5359 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5360 5361 /* compute doesn't have PFP */ 5362 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5363 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5364 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5365 amdgpu_ring_write(ring, 0x0); 5366 } 5367 } 5368 5369 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5370 { 5371 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5372 } 5373 5374 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5375 { 5376 u64 wptr; 5377 5378 /* XXX check if swapping is necessary on BE */ 5379 if (ring->use_doorbell) 5380 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5381 else 5382 BUG(); 5383 return wptr; 5384 } 5385 5386 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5387 { 5388 struct amdgpu_device *adev = ring->adev; 5389 5390 /* XXX check if swapping is necessary on BE */ 5391 if (ring->use_doorbell) { 5392 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5393 WDOORBELL64(ring->doorbell_index, ring->wptr); 5394 } else{ 5395 BUG(); /* only DOORBELL method supported on gfx9 now */ 5396 } 5397 } 5398 5399 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5400 u64 seq, unsigned int flags) 5401 { 5402 struct amdgpu_device *adev = ring->adev; 5403 5404 /* we only allocate 32bit for each seq wb address */ 5405 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5406 5407 /* write fence seq to the "addr" */ 5408 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5409 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5410 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5411 amdgpu_ring_write(ring, lower_32_bits(addr)); 5412 amdgpu_ring_write(ring, upper_32_bits(addr)); 5413 amdgpu_ring_write(ring, lower_32_bits(seq)); 5414 5415 if (flags & AMDGPU_FENCE_FLAG_INT) { 5416 /* set register to trigger INT */ 5417 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5418 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5419 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5420 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5421 amdgpu_ring_write(ring, 0); 5422 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5423 } 5424 } 5425 5426 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5427 { 5428 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5429 amdgpu_ring_write(ring, 0); 5430 } 5431 5432 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5433 { 5434 struct v9_ce_ib_state ce_payload = {0}; 5435 uint64_t csa_addr; 5436 int cnt; 5437 5438 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5439 csa_addr = amdgpu_csa_vaddr(ring->adev); 5440 5441 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5442 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5443 WRITE_DATA_DST_SEL(8) | 5444 WR_CONFIRM) | 5445 WRITE_DATA_CACHE_POLICY(0)); 5446 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5447 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5448 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5449 } 5450 5451 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5452 { 5453 struct v9_de_ib_state de_payload = {0}; 5454 uint64_t csa_addr, gds_addr; 5455 int cnt; 5456 5457 csa_addr = amdgpu_csa_vaddr(ring->adev); 5458 gds_addr = csa_addr + 4096; 5459 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5460 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5461 5462 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5463 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5464 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5465 WRITE_DATA_DST_SEL(8) | 5466 WR_CONFIRM) | 5467 WRITE_DATA_CACHE_POLICY(0)); 5468 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5469 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5470 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5471 } 5472 5473 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5474 bool secure) 5475 { 5476 uint32_t v = secure ? FRAME_TMZ : 0; 5477 5478 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5479 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5480 } 5481 5482 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5483 { 5484 uint32_t dw2 = 0; 5485 5486 if (amdgpu_sriov_vf(ring->adev)) 5487 gfx_v9_0_ring_emit_ce_meta(ring); 5488 5489 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5490 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5491 /* set load_global_config & load_global_uconfig */ 5492 dw2 |= 0x8001; 5493 /* set load_cs_sh_regs */ 5494 dw2 |= 0x01000000; 5495 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5496 dw2 |= 0x10002; 5497 5498 /* set load_ce_ram if preamble presented */ 5499 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5500 dw2 |= 0x10000000; 5501 } else { 5502 /* still load_ce_ram if this is the first time preamble presented 5503 * although there is no context switch happens. 5504 */ 5505 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5506 dw2 |= 0x10000000; 5507 } 5508 5509 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5510 amdgpu_ring_write(ring, dw2); 5511 amdgpu_ring_write(ring, 0); 5512 } 5513 5514 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5515 { 5516 unsigned ret; 5517 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5518 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5519 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5520 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5521 ret = ring->wptr & ring->buf_mask; 5522 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5523 return ret; 5524 } 5525 5526 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5527 { 5528 unsigned cur; 5529 BUG_ON(offset > ring->buf_mask); 5530 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5531 5532 cur = (ring->wptr & ring->buf_mask) - 1; 5533 if (likely(cur > offset)) 5534 ring->ring[offset] = cur - offset; 5535 else 5536 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5537 } 5538 5539 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5540 uint32_t reg_val_offs) 5541 { 5542 struct amdgpu_device *adev = ring->adev; 5543 5544 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5545 amdgpu_ring_write(ring, 0 | /* src: register*/ 5546 (5 << 8) | /* dst: memory */ 5547 (1 << 20)); /* write confirm */ 5548 amdgpu_ring_write(ring, reg); 5549 amdgpu_ring_write(ring, 0); 5550 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5551 reg_val_offs * 4)); 5552 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5553 reg_val_offs * 4)); 5554 } 5555 5556 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5557 uint32_t val) 5558 { 5559 uint32_t cmd = 0; 5560 5561 switch (ring->funcs->type) { 5562 case AMDGPU_RING_TYPE_GFX: 5563 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5564 break; 5565 case AMDGPU_RING_TYPE_KIQ: 5566 cmd = (1 << 16); /* no inc addr */ 5567 break; 5568 default: 5569 cmd = WR_CONFIRM; 5570 break; 5571 } 5572 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5573 amdgpu_ring_write(ring, cmd); 5574 amdgpu_ring_write(ring, reg); 5575 amdgpu_ring_write(ring, 0); 5576 amdgpu_ring_write(ring, val); 5577 } 5578 5579 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5580 uint32_t val, uint32_t mask) 5581 { 5582 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5583 } 5584 5585 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5586 uint32_t reg0, uint32_t reg1, 5587 uint32_t ref, uint32_t mask) 5588 { 5589 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5590 struct amdgpu_device *adev = ring->adev; 5591 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5592 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5593 5594 if (fw_version_ok) 5595 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5596 ref, mask, 0x20); 5597 else 5598 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5599 ref, mask); 5600 } 5601 5602 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5603 { 5604 struct amdgpu_device *adev = ring->adev; 5605 uint32_t value = 0; 5606 5607 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5608 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5609 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5610 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5611 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5612 } 5613 5614 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5615 enum amdgpu_interrupt_state state) 5616 { 5617 switch (state) { 5618 case AMDGPU_IRQ_STATE_DISABLE: 5619 case AMDGPU_IRQ_STATE_ENABLE: 5620 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5621 TIME_STAMP_INT_ENABLE, 5622 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5623 break; 5624 default: 5625 break; 5626 } 5627 } 5628 5629 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5630 int me, int pipe, 5631 enum amdgpu_interrupt_state state) 5632 { 5633 u32 mec_int_cntl, mec_int_cntl_reg; 5634 5635 /* 5636 * amdgpu controls only the first MEC. That's why this function only 5637 * handles the setting of interrupts for this specific MEC. All other 5638 * pipes' interrupts are set by amdkfd. 5639 */ 5640 5641 if (me == 1) { 5642 switch (pipe) { 5643 case 0: 5644 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5645 break; 5646 case 1: 5647 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5648 break; 5649 case 2: 5650 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5651 break; 5652 case 3: 5653 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5654 break; 5655 default: 5656 DRM_DEBUG("invalid pipe %d\n", pipe); 5657 return; 5658 } 5659 } else { 5660 DRM_DEBUG("invalid me %d\n", me); 5661 return; 5662 } 5663 5664 switch (state) { 5665 case AMDGPU_IRQ_STATE_DISABLE: 5666 mec_int_cntl = RREG32(mec_int_cntl_reg); 5667 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5668 TIME_STAMP_INT_ENABLE, 0); 5669 WREG32(mec_int_cntl_reg, mec_int_cntl); 5670 break; 5671 case AMDGPU_IRQ_STATE_ENABLE: 5672 mec_int_cntl = RREG32(mec_int_cntl_reg); 5673 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5674 TIME_STAMP_INT_ENABLE, 1); 5675 WREG32(mec_int_cntl_reg, mec_int_cntl); 5676 break; 5677 default: 5678 break; 5679 } 5680 } 5681 5682 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5683 struct amdgpu_irq_src *source, 5684 unsigned type, 5685 enum amdgpu_interrupt_state state) 5686 { 5687 switch (state) { 5688 case AMDGPU_IRQ_STATE_DISABLE: 5689 case AMDGPU_IRQ_STATE_ENABLE: 5690 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5691 PRIV_REG_INT_ENABLE, 5692 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5693 break; 5694 default: 5695 break; 5696 } 5697 5698 return 0; 5699 } 5700 5701 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5702 struct amdgpu_irq_src *source, 5703 unsigned type, 5704 enum amdgpu_interrupt_state state) 5705 { 5706 switch (state) { 5707 case AMDGPU_IRQ_STATE_DISABLE: 5708 case AMDGPU_IRQ_STATE_ENABLE: 5709 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5710 PRIV_INSTR_INT_ENABLE, 5711 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5712 default: 5713 break; 5714 } 5715 5716 return 0; 5717 } 5718 5719 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5720 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5721 CP_ECC_ERROR_INT_ENABLE, 1) 5722 5723 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5724 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5725 CP_ECC_ERROR_INT_ENABLE, 0) 5726 5727 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5728 struct amdgpu_irq_src *source, 5729 unsigned type, 5730 enum amdgpu_interrupt_state state) 5731 { 5732 switch (state) { 5733 case AMDGPU_IRQ_STATE_DISABLE: 5734 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5735 CP_ECC_ERROR_INT_ENABLE, 0); 5736 DISABLE_ECC_ON_ME_PIPE(1, 0); 5737 DISABLE_ECC_ON_ME_PIPE(1, 1); 5738 DISABLE_ECC_ON_ME_PIPE(1, 2); 5739 DISABLE_ECC_ON_ME_PIPE(1, 3); 5740 break; 5741 5742 case AMDGPU_IRQ_STATE_ENABLE: 5743 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5744 CP_ECC_ERROR_INT_ENABLE, 1); 5745 ENABLE_ECC_ON_ME_PIPE(1, 0); 5746 ENABLE_ECC_ON_ME_PIPE(1, 1); 5747 ENABLE_ECC_ON_ME_PIPE(1, 2); 5748 ENABLE_ECC_ON_ME_PIPE(1, 3); 5749 break; 5750 default: 5751 break; 5752 } 5753 5754 return 0; 5755 } 5756 5757 5758 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5759 struct amdgpu_irq_src *src, 5760 unsigned type, 5761 enum amdgpu_interrupt_state state) 5762 { 5763 switch (type) { 5764 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5765 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5766 break; 5767 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5768 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5769 break; 5770 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5771 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5772 break; 5773 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5774 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5775 break; 5776 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5777 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5778 break; 5779 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5780 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5781 break; 5782 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5783 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5784 break; 5785 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5786 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5787 break; 5788 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5789 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5790 break; 5791 default: 5792 break; 5793 } 5794 return 0; 5795 } 5796 5797 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5798 struct amdgpu_irq_src *source, 5799 struct amdgpu_iv_entry *entry) 5800 { 5801 int i; 5802 u8 me_id, pipe_id, queue_id; 5803 struct amdgpu_ring *ring; 5804 5805 DRM_DEBUG("IH: CP EOP\n"); 5806 me_id = (entry->ring_id & 0x0c) >> 2; 5807 pipe_id = (entry->ring_id & 0x03) >> 0; 5808 queue_id = (entry->ring_id & 0x70) >> 4; 5809 5810 switch (me_id) { 5811 case 0: 5812 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5813 break; 5814 case 1: 5815 case 2: 5816 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5817 ring = &adev->gfx.compute_ring[i]; 5818 /* Per-queue interrupt is supported for MEC starting from VI. 5819 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5820 */ 5821 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5822 amdgpu_fence_process(ring); 5823 } 5824 break; 5825 } 5826 return 0; 5827 } 5828 5829 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5830 struct amdgpu_iv_entry *entry) 5831 { 5832 u8 me_id, pipe_id, queue_id; 5833 struct amdgpu_ring *ring; 5834 int i; 5835 5836 me_id = (entry->ring_id & 0x0c) >> 2; 5837 pipe_id = (entry->ring_id & 0x03) >> 0; 5838 queue_id = (entry->ring_id & 0x70) >> 4; 5839 5840 switch (me_id) { 5841 case 0: 5842 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5843 break; 5844 case 1: 5845 case 2: 5846 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5847 ring = &adev->gfx.compute_ring[i]; 5848 if (ring->me == me_id && ring->pipe == pipe_id && 5849 ring->queue == queue_id) 5850 drm_sched_fault(&ring->sched); 5851 } 5852 break; 5853 } 5854 } 5855 5856 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5857 struct amdgpu_irq_src *source, 5858 struct amdgpu_iv_entry *entry) 5859 { 5860 DRM_ERROR("Illegal register access in command stream\n"); 5861 gfx_v9_0_fault(adev, entry); 5862 return 0; 5863 } 5864 5865 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5866 struct amdgpu_irq_src *source, 5867 struct amdgpu_iv_entry *entry) 5868 { 5869 DRM_ERROR("Illegal instruction in command stream\n"); 5870 gfx_v9_0_fault(adev, entry); 5871 return 0; 5872 } 5873 5874 5875 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5876 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5877 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5878 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5879 }, 5880 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5881 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5882 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5883 }, 5884 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5885 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5886 0, 0 5887 }, 5888 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5889 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5890 0, 0 5891 }, 5892 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5893 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5894 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5895 }, 5896 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5897 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5898 0, 0 5899 }, 5900 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5901 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5902 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5903 }, 5904 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5905 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5906 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5907 }, 5908 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5909 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5910 0, 0 5911 }, 5912 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5913 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5914 0, 0 5915 }, 5916 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5917 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5918 0, 0 5919 }, 5920 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5921 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5922 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5923 }, 5924 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5925 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5926 0, 0 5927 }, 5928 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5929 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5930 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5931 }, 5932 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5933 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5934 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5935 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5936 }, 5937 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 5938 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5939 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 5940 0, 0 5941 }, 5942 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 5943 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5944 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 5945 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 5946 }, 5947 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 5948 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5949 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 5950 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 5951 }, 5952 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 5953 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5954 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 5955 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 5956 }, 5957 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 5958 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5959 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 5960 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 5961 }, 5962 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 5963 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 5964 0, 0 5965 }, 5966 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5967 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 5968 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 5969 }, 5970 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5971 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 5972 0, 0 5973 }, 5974 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5975 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 5976 0, 0 5977 }, 5978 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5979 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 5980 0, 0 5981 }, 5982 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5983 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 5984 0, 0 5985 }, 5986 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5987 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 5988 0, 0 5989 }, 5990 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5991 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 5992 0, 0 5993 }, 5994 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5995 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 5996 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 5997 }, 5998 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5999 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6000 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6001 }, 6002 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6003 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6004 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6005 }, 6006 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6007 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6008 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6009 }, 6010 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6011 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6012 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6013 }, 6014 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6015 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6016 0, 0 6017 }, 6018 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6019 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6020 0, 0 6021 }, 6022 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6023 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6024 0, 0 6025 }, 6026 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6027 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6028 0, 0 6029 }, 6030 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6031 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6032 0, 0 6033 }, 6034 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6035 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6036 0, 0 6037 }, 6038 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6039 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6040 0, 0 6041 }, 6042 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6043 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6044 0, 0 6045 }, 6046 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6047 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6048 0, 0 6049 }, 6050 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6051 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6052 0, 0 6053 }, 6054 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6055 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6056 0, 0 6057 }, 6058 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6059 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6060 0, 0 6061 }, 6062 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6063 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6064 0, 0 6065 }, 6066 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6067 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6068 0, 0 6069 }, 6070 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6071 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6072 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6073 }, 6074 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6075 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6076 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6077 }, 6078 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6079 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6080 0, 0 6081 }, 6082 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6083 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6084 0, 0 6085 }, 6086 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6087 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6088 0, 0 6089 }, 6090 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6091 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6092 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6093 }, 6094 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6095 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6096 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6097 }, 6098 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6099 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6100 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6101 }, 6102 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6103 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6104 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6105 }, 6106 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6107 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6108 0, 0 6109 }, 6110 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6111 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6112 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6113 }, 6114 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6115 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6116 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6117 }, 6118 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6119 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6120 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6121 }, 6122 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6123 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6124 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6125 }, 6126 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6127 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6128 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6129 }, 6130 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6131 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6132 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6133 }, 6134 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6135 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6136 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6137 }, 6138 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6139 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6140 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6141 }, 6142 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6143 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6144 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6145 }, 6146 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6147 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6148 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6149 }, 6150 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6151 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6152 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6153 }, 6154 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6155 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6156 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6157 }, 6158 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6159 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6160 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6161 }, 6162 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6163 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6164 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6165 }, 6166 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6167 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6168 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6169 }, 6170 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6171 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6172 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6173 }, 6174 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6175 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6176 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6177 }, 6178 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6179 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6180 0, 0 6181 }, 6182 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6183 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6184 0, 0 6185 }, 6186 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6187 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6188 0, 0 6189 }, 6190 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6191 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6192 0, 0 6193 }, 6194 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6195 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6196 0, 0 6197 }, 6198 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6199 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6200 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6201 }, 6202 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6203 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6204 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6205 }, 6206 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6207 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6208 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6209 }, 6210 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6211 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6212 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6213 }, 6214 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6215 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6216 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6217 }, 6218 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6219 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6220 0, 0 6221 }, 6222 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6223 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6224 0, 0 6225 }, 6226 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6227 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6228 0, 0 6229 }, 6230 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6231 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6232 0, 0 6233 }, 6234 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6235 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6236 0, 0 6237 }, 6238 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6239 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6240 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6241 }, 6242 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6243 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6244 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6245 }, 6246 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6247 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6248 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6249 }, 6250 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6251 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6252 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6253 }, 6254 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6255 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6256 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6257 }, 6258 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6259 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6260 0, 0 6261 }, 6262 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6263 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6264 0, 0 6265 }, 6266 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6267 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6268 0, 0 6269 }, 6270 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6271 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6272 0, 0 6273 }, 6274 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6275 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6276 0, 0 6277 }, 6278 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6279 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6280 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6281 }, 6282 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6283 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6284 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6285 }, 6286 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6287 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6288 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6289 }, 6290 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6291 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6292 0, 0 6293 }, 6294 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6295 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6296 0, 0 6297 }, 6298 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6299 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6300 0, 0 6301 }, 6302 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6303 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6304 0, 0 6305 }, 6306 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6307 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6308 0, 0 6309 }, 6310 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6311 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6312 0, 0 6313 } 6314 }; 6315 6316 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6317 void *inject_if) 6318 { 6319 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6320 int ret; 6321 struct ta_ras_trigger_error_input block_info = { 0 }; 6322 6323 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6324 return -EINVAL; 6325 6326 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6327 return -EINVAL; 6328 6329 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6330 return -EPERM; 6331 6332 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6333 info->head.type)) { 6334 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6335 ras_gfx_subblocks[info->head.sub_block_index].name, 6336 info->head.type); 6337 return -EPERM; 6338 } 6339 6340 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6341 info->head.type)) { 6342 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6343 ras_gfx_subblocks[info->head.sub_block_index].name, 6344 info->head.type); 6345 return -EPERM; 6346 } 6347 6348 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6349 block_info.sub_block_index = 6350 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6351 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6352 block_info.address = info->address; 6353 block_info.value = info->value; 6354 6355 mutex_lock(&adev->grbm_idx_mutex); 6356 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6357 mutex_unlock(&adev->grbm_idx_mutex); 6358 6359 return ret; 6360 } 6361 6362 static const char *vml2_mems[] = { 6363 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6364 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6365 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6366 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6367 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6368 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6369 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6370 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6371 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6372 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6373 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6374 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6375 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6376 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6377 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6378 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6379 }; 6380 6381 static const char *vml2_walker_mems[] = { 6382 "UTC_VML2_CACHE_PDE0_MEM0", 6383 "UTC_VML2_CACHE_PDE0_MEM1", 6384 "UTC_VML2_CACHE_PDE1_MEM0", 6385 "UTC_VML2_CACHE_PDE1_MEM1", 6386 "UTC_VML2_CACHE_PDE2_MEM0", 6387 "UTC_VML2_CACHE_PDE2_MEM1", 6388 "UTC_VML2_RDIF_LOG_FIFO", 6389 }; 6390 6391 static const char *atc_l2_cache_2m_mems[] = { 6392 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6393 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6394 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6395 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6396 }; 6397 6398 static const char *atc_l2_cache_4k_mems[] = { 6399 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6400 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6401 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6402 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6403 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6404 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6405 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6406 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6407 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6408 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6409 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6410 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6411 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6412 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6413 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6414 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6415 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6416 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6417 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6418 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6419 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6420 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6421 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6422 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6423 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6424 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6425 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6426 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6427 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6428 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6429 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6430 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6431 }; 6432 6433 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6434 struct ras_err_data *err_data) 6435 { 6436 uint32_t i, data; 6437 uint32_t sec_count, ded_count; 6438 6439 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6440 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6441 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6442 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6443 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6444 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6445 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6446 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6447 6448 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6449 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6450 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6451 6452 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6453 if (sec_count) { 6454 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6455 "SEC %d\n", i, vml2_mems[i], sec_count); 6456 err_data->ce_count += sec_count; 6457 } 6458 6459 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6460 if (ded_count) { 6461 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6462 "DED %d\n", i, vml2_mems[i], ded_count); 6463 err_data->ue_count += ded_count; 6464 } 6465 } 6466 6467 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6468 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6469 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6470 6471 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6472 SEC_COUNT); 6473 if (sec_count) { 6474 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6475 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6476 err_data->ce_count += sec_count; 6477 } 6478 6479 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6480 DED_COUNT); 6481 if (ded_count) { 6482 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6483 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6484 err_data->ue_count += ded_count; 6485 } 6486 } 6487 6488 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6489 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6490 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6491 6492 sec_count = (data & 0x00006000L) >> 0xd; 6493 if (sec_count) { 6494 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6495 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6496 sec_count); 6497 err_data->ce_count += sec_count; 6498 } 6499 } 6500 6501 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6502 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6503 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6504 6505 sec_count = (data & 0x00006000L) >> 0xd; 6506 if (sec_count) { 6507 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6508 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6509 sec_count); 6510 err_data->ce_count += sec_count; 6511 } 6512 6513 ded_count = (data & 0x00018000L) >> 0xf; 6514 if (ded_count) { 6515 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6516 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6517 ded_count); 6518 err_data->ue_count += ded_count; 6519 } 6520 } 6521 6522 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6523 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6524 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6525 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6526 6527 return 0; 6528 } 6529 6530 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6531 const struct soc15_reg_entry *reg, 6532 uint32_t se_id, uint32_t inst_id, uint32_t value, 6533 uint32_t *sec_count, uint32_t *ded_count) 6534 { 6535 uint32_t i; 6536 uint32_t sec_cnt, ded_cnt; 6537 6538 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6539 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6540 gfx_v9_0_ras_fields[i].seg != reg->seg || 6541 gfx_v9_0_ras_fields[i].inst != reg->inst) 6542 continue; 6543 6544 sec_cnt = (value & 6545 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6546 gfx_v9_0_ras_fields[i].sec_count_shift; 6547 if (sec_cnt) { 6548 dev_info(adev->dev, "GFX SubBlock %s, " 6549 "Instance[%d][%d], SEC %d\n", 6550 gfx_v9_0_ras_fields[i].name, 6551 se_id, inst_id, 6552 sec_cnt); 6553 *sec_count += sec_cnt; 6554 } 6555 6556 ded_cnt = (value & 6557 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6558 gfx_v9_0_ras_fields[i].ded_count_shift; 6559 if (ded_cnt) { 6560 dev_info(adev->dev, "GFX SubBlock %s, " 6561 "Instance[%d][%d], DED %d\n", 6562 gfx_v9_0_ras_fields[i].name, 6563 se_id, inst_id, 6564 ded_cnt); 6565 *ded_count += ded_cnt; 6566 } 6567 } 6568 6569 return 0; 6570 } 6571 6572 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6573 { 6574 int i, j, k; 6575 6576 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6577 return; 6578 6579 /* read back registers to clear the counters */ 6580 mutex_lock(&adev->grbm_idx_mutex); 6581 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6582 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6583 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6584 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6585 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6586 } 6587 } 6588 } 6589 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6590 mutex_unlock(&adev->grbm_idx_mutex); 6591 6592 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6593 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6594 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6595 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6596 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6597 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6598 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6599 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6600 6601 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6602 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6603 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6604 } 6605 6606 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6607 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6608 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6609 } 6610 6611 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6612 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6613 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6614 } 6615 6616 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6617 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6618 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6619 } 6620 6621 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6622 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6623 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6624 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6625 } 6626 6627 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6628 void *ras_error_status) 6629 { 6630 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6631 uint32_t sec_count = 0, ded_count = 0; 6632 uint32_t i, j, k; 6633 uint32_t reg_value; 6634 6635 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6636 return -EINVAL; 6637 6638 err_data->ue_count = 0; 6639 err_data->ce_count = 0; 6640 6641 mutex_lock(&adev->grbm_idx_mutex); 6642 6643 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6644 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6645 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6646 gfx_v9_0_select_se_sh(adev, j, 0, k); 6647 reg_value = 6648 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6649 if (reg_value) 6650 gfx_v9_0_ras_error_count(adev, 6651 &gfx_v9_0_edc_counter_regs[i], 6652 j, k, reg_value, 6653 &sec_count, &ded_count); 6654 } 6655 } 6656 } 6657 6658 err_data->ce_count += sec_count; 6659 err_data->ue_count += ded_count; 6660 6661 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6662 mutex_unlock(&adev->grbm_idx_mutex); 6663 6664 gfx_v9_0_query_utc_edc_status(adev, err_data); 6665 6666 return 0; 6667 } 6668 6669 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6670 { 6671 const unsigned int cp_coher_cntl = 6672 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6673 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6674 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6675 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6676 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6677 6678 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6679 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6680 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6681 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6682 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6683 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6684 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6685 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6686 } 6687 6688 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6689 .name = "gfx_v9_0", 6690 .early_init = gfx_v9_0_early_init, 6691 .late_init = gfx_v9_0_late_init, 6692 .sw_init = gfx_v9_0_sw_init, 6693 .sw_fini = gfx_v9_0_sw_fini, 6694 .hw_init = gfx_v9_0_hw_init, 6695 .hw_fini = gfx_v9_0_hw_fini, 6696 .suspend = gfx_v9_0_suspend, 6697 .resume = gfx_v9_0_resume, 6698 .is_idle = gfx_v9_0_is_idle, 6699 .wait_for_idle = gfx_v9_0_wait_for_idle, 6700 .soft_reset = gfx_v9_0_soft_reset, 6701 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6702 .set_powergating_state = gfx_v9_0_set_powergating_state, 6703 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6704 }; 6705 6706 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6707 .type = AMDGPU_RING_TYPE_GFX, 6708 .align_mask = 0xff, 6709 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6710 .support_64bit_ptrs = true, 6711 .vmhub = AMDGPU_GFXHUB_0, 6712 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6713 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6714 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6715 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6716 5 + /* COND_EXEC */ 6717 7 + /* PIPELINE_SYNC */ 6718 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6719 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6720 2 + /* VM_FLUSH */ 6721 8 + /* FENCE for VM_FLUSH */ 6722 20 + /* GDS switch */ 6723 4 + /* double SWITCH_BUFFER, 6724 the first COND_EXEC jump to the place just 6725 prior to this double SWITCH_BUFFER */ 6726 5 + /* COND_EXEC */ 6727 7 + /* HDP_flush */ 6728 4 + /* VGT_flush */ 6729 14 + /* CE_META */ 6730 31 + /* DE_META */ 6731 3 + /* CNTX_CTRL */ 6732 5 + /* HDP_INVL */ 6733 8 + 8 + /* FENCE x2 */ 6734 2 + /* SWITCH_BUFFER */ 6735 7, /* gfx_v9_0_emit_mem_sync */ 6736 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6737 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6738 .emit_fence = gfx_v9_0_ring_emit_fence, 6739 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6740 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6741 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6742 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6743 .test_ring = gfx_v9_0_ring_test_ring, 6744 .test_ib = gfx_v9_0_ring_test_ib, 6745 .insert_nop = amdgpu_ring_insert_nop, 6746 .pad_ib = amdgpu_ring_generic_pad_ib, 6747 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6748 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6749 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6750 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6751 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6752 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6753 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6754 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6755 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6756 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6757 }; 6758 6759 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6760 .type = AMDGPU_RING_TYPE_COMPUTE, 6761 .align_mask = 0xff, 6762 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6763 .support_64bit_ptrs = true, 6764 .vmhub = AMDGPU_GFXHUB_0, 6765 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6766 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6767 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6768 .emit_frame_size = 6769 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6770 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6771 5 + /* hdp invalidate */ 6772 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6773 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6774 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6775 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6776 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6777 7, /* gfx_v9_0_emit_mem_sync */ 6778 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6779 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6780 .emit_fence = gfx_v9_0_ring_emit_fence, 6781 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6782 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6783 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6784 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6785 .test_ring = gfx_v9_0_ring_test_ring, 6786 .test_ib = gfx_v9_0_ring_test_ib, 6787 .insert_nop = amdgpu_ring_insert_nop, 6788 .pad_ib = amdgpu_ring_generic_pad_ib, 6789 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6790 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6791 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6792 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6793 }; 6794 6795 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6796 .type = AMDGPU_RING_TYPE_KIQ, 6797 .align_mask = 0xff, 6798 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6799 .support_64bit_ptrs = true, 6800 .vmhub = AMDGPU_GFXHUB_0, 6801 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6802 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6803 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6804 .emit_frame_size = 6805 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6806 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6807 5 + /* hdp invalidate */ 6808 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6809 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6810 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6811 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6812 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6813 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6814 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6815 .test_ring = gfx_v9_0_ring_test_ring, 6816 .insert_nop = amdgpu_ring_insert_nop, 6817 .pad_ib = amdgpu_ring_generic_pad_ib, 6818 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6819 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6820 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6821 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6822 }; 6823 6824 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6825 { 6826 int i; 6827 6828 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6829 6830 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6831 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6832 6833 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6834 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6835 } 6836 6837 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6838 .set = gfx_v9_0_set_eop_interrupt_state, 6839 .process = gfx_v9_0_eop_irq, 6840 }; 6841 6842 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6843 .set = gfx_v9_0_set_priv_reg_fault_state, 6844 .process = gfx_v9_0_priv_reg_irq, 6845 }; 6846 6847 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6848 .set = gfx_v9_0_set_priv_inst_fault_state, 6849 .process = gfx_v9_0_priv_inst_irq, 6850 }; 6851 6852 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6853 .set = gfx_v9_0_set_cp_ecc_error_state, 6854 .process = amdgpu_gfx_cp_ecc_error_irq, 6855 }; 6856 6857 6858 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6859 { 6860 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6861 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6862 6863 adev->gfx.priv_reg_irq.num_types = 1; 6864 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6865 6866 adev->gfx.priv_inst_irq.num_types = 1; 6867 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6868 6869 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6870 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6871 } 6872 6873 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6874 { 6875 switch (adev->asic_type) { 6876 case CHIP_VEGA10: 6877 case CHIP_VEGA12: 6878 case CHIP_VEGA20: 6879 case CHIP_RAVEN: 6880 case CHIP_ARCTURUS: 6881 case CHIP_RENOIR: 6882 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 6883 break; 6884 default: 6885 break; 6886 } 6887 } 6888 6889 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 6890 { 6891 /* init asci gds info */ 6892 switch (adev->asic_type) { 6893 case CHIP_VEGA10: 6894 case CHIP_VEGA12: 6895 case CHIP_VEGA20: 6896 adev->gds.gds_size = 0x10000; 6897 break; 6898 case CHIP_RAVEN: 6899 case CHIP_ARCTURUS: 6900 adev->gds.gds_size = 0x1000; 6901 break; 6902 default: 6903 adev->gds.gds_size = 0x10000; 6904 break; 6905 } 6906 6907 switch (adev->asic_type) { 6908 case CHIP_VEGA10: 6909 case CHIP_VEGA20: 6910 adev->gds.gds_compute_max_wave_id = 0x7ff; 6911 break; 6912 case CHIP_VEGA12: 6913 adev->gds.gds_compute_max_wave_id = 0x27f; 6914 break; 6915 case CHIP_RAVEN: 6916 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 6917 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 6918 else 6919 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 6920 break; 6921 case CHIP_ARCTURUS: 6922 adev->gds.gds_compute_max_wave_id = 0xfff; 6923 break; 6924 default: 6925 /* this really depends on the chip */ 6926 adev->gds.gds_compute_max_wave_id = 0x7ff; 6927 break; 6928 } 6929 6930 adev->gds.gws_size = 64; 6931 adev->gds.oa_size = 16; 6932 } 6933 6934 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6935 u32 bitmap) 6936 { 6937 u32 data; 6938 6939 if (!bitmap) 6940 return; 6941 6942 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6943 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6944 6945 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 6946 } 6947 6948 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6949 { 6950 u32 data, mask; 6951 6952 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 6953 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 6954 6955 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6956 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6957 6958 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6959 6960 return (~data) & mask; 6961 } 6962 6963 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 6964 struct amdgpu_cu_info *cu_info) 6965 { 6966 int i, j, k, counter, active_cu_number = 0; 6967 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6968 unsigned disable_masks[4 * 4]; 6969 6970 if (!adev || !cu_info) 6971 return -EINVAL; 6972 6973 /* 6974 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 6975 */ 6976 if (adev->gfx.config.max_shader_engines * 6977 adev->gfx.config.max_sh_per_se > 16) 6978 return -EINVAL; 6979 6980 amdgpu_gfx_parse_disable_cu(disable_masks, 6981 adev->gfx.config.max_shader_engines, 6982 adev->gfx.config.max_sh_per_se); 6983 6984 mutex_lock(&adev->grbm_idx_mutex); 6985 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6986 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 6987 mask = 1; 6988 ao_bitmap = 0; 6989 counter = 0; 6990 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 6991 gfx_v9_0_set_user_cu_inactive_bitmap( 6992 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 6993 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 6994 6995 /* 6996 * The bitmap(and ao_cu_bitmap) in cu_info structure is 6997 * 4x4 size array, and it's usually suitable for Vega 6998 * ASICs which has 4*2 SE/SH layout. 6999 * But for Arcturus, SE/SH layout is changed to 8*1. 7000 * To mostly reduce the impact, we make it compatible 7001 * with current bitmap array as below: 7002 * SE4,SH0 --> bitmap[0][1] 7003 * SE5,SH0 --> bitmap[1][1] 7004 * SE6,SH0 --> bitmap[2][1] 7005 * SE7,SH0 --> bitmap[3][1] 7006 */ 7007 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7008 7009 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7010 if (bitmap & mask) { 7011 if (counter < adev->gfx.config.max_cu_per_sh) 7012 ao_bitmap |= mask; 7013 counter ++; 7014 } 7015 mask <<= 1; 7016 } 7017 active_cu_number += counter; 7018 if (i < 2 && j < 2) 7019 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7020 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7021 } 7022 } 7023 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7024 mutex_unlock(&adev->grbm_idx_mutex); 7025 7026 cu_info->number = active_cu_number; 7027 cu_info->ao_cu_mask = ao_cu_mask; 7028 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7029 7030 return 0; 7031 } 7032 7033 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7034 { 7035 .type = AMD_IP_BLOCK_TYPE_GFX, 7036 .major = 9, 7037 .minor = 0, 7038 .rev = 0, 7039 .funcs = &gfx_v9_0_ip_funcs, 7040 }; 7041