1 /* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 * 24 * $FreeBSD: head/sys/dev/drm2/radeon/evergreen.c 254885 2013-08-25 19:37:15Z dumbbell $ 25 */ 26 27 #include <drm/drmP.h> 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include <uapi_drm/radeon_drm.h> 31 #include "evergreend.h" 32 #include "atom.h" 33 #include "avivod.h" 34 #include "evergreen_reg.h" 35 #include "evergreen_blit_shaders.h" 36 37 #define EVERGREEN_PFP_UCODE_SIZE 1120 38 #define EVERGREEN_PM4_UCODE_SIZE 1376 39 40 static const u32 crtc_offsets[6] = 41 { 42 EVERGREEN_CRTC0_REGISTER_OFFSET, 43 EVERGREEN_CRTC1_REGISTER_OFFSET, 44 EVERGREEN_CRTC2_REGISTER_OFFSET, 45 EVERGREEN_CRTC3_REGISTER_OFFSET, 46 EVERGREEN_CRTC4_REGISTER_OFFSET, 47 EVERGREEN_CRTC5_REGISTER_OFFSET 48 }; 49 50 static void evergreen_gpu_init(struct radeon_device *rdev); 51 void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 52 void evergreen_print_gpu_status_regs(struct radeon_device *rdev); 53 bool evergreen_is_display_hung(struct radeon_device *rdev); 54 55 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, 56 unsigned *bankh, unsigned *mtaspect, 57 unsigned *tile_split) 58 { 59 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 60 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 61 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 62 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 63 switch (*bankw) { 64 default: 65 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break; 66 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break; 67 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break; 68 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break; 69 } 70 switch (*bankh) { 71 default: 72 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break; 73 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break; 74 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break; 75 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break; 76 } 77 switch (*mtaspect) { 78 default: 79 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break; 80 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break; 81 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break; 82 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break; 83 } 84 } 85 86 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 87 { 88 u16 ctl, v; 89 int err, cap; 90 91 err = pci_find_extcap(rdev->dev, PCIY_EXPRESS, &cap); 92 if (err) 93 return; 94 95 cap += PCIER_DEVCTRL; 96 97 ctl = pci_read_config(rdev->dev, cap, 2); 98 99 v = (ctl & PCIEM_DEVCTL_MAX_READRQ_MASK) >> 12; 100 101 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it 102 * to avoid hangs or perfomance issues 103 */ 104 if ((v == 0) || (v == 6) || (v == 7)) { 105 ctl &= ~PCIEM_DEVCTL_MAX_READRQ_MASK; 106 ctl |= (2 << 12); 107 pci_write_config(rdev->dev, cap, ctl, 2); 108 } 109 } 110 111 /** 112 * dce4_wait_for_vblank - vblank wait asic callback. 113 * 114 * @rdev: radeon_device pointer 115 * @crtc: crtc to wait for vblank on 116 * 117 * Wait for vblank on the requested crtc (evergreen+). 118 */ 119 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) 120 { 121 int i; 122 123 if (crtc >= rdev->num_crtc) 124 return; 125 126 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) { 127 for (i = 0; i < rdev->usec_timeout; i++) { 128 if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)) 129 break; 130 DRM_UDELAY(1); 131 } 132 for (i = 0; i < rdev->usec_timeout; i++) { 133 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK) 134 break; 135 DRM_UDELAY(1); 136 } 137 } 138 } 139 140 /** 141 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback. 142 * 143 * @rdev: radeon_device pointer 144 * @crtc: crtc to prepare for pageflip on 145 * 146 * Pre-pageflip callback (evergreen+). 147 * Enables the pageflip irq (vblank irq). 148 */ 149 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc) 150 { 151 /* enable the pflip int */ 152 radeon_irq_kms_pflip_irq_get(rdev, crtc); 153 } 154 155 /** 156 * evergreen_post_page_flip - pos-pageflip callback. 157 * 158 * @rdev: radeon_device pointer 159 * @crtc: crtc to cleanup pageflip on 160 * 161 * Post-pageflip callback (evergreen+). 162 * Disables the pageflip irq (vblank irq). 163 */ 164 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc) 165 { 166 /* disable the pflip int */ 167 radeon_irq_kms_pflip_irq_put(rdev, crtc); 168 } 169 170 /** 171 * evergreen_page_flip - pageflip callback. 172 * 173 * @rdev: radeon_device pointer 174 * @crtc_id: crtc to cleanup pageflip on 175 * @crtc_base: new address of the crtc (GPU MC address) 176 * 177 * Does the actual pageflip (evergreen+). 178 * During vblank we take the crtc lock and wait for the update_pending 179 * bit to go high, when it does, we release the lock, and allow the 180 * double buffered update to take place. 181 * Returns the current update pending status. 182 */ 183 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) 184 { 185 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 186 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset); 187 int i; 188 189 /* Lock the graphics update lock */ 190 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 191 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 192 193 /* update the scanout addresses */ 194 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 195 upper_32_bits(crtc_base)); 196 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 197 (u32)crtc_base); 198 199 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 200 upper_32_bits(crtc_base)); 201 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 202 (u32)crtc_base); 203 204 /* Wait for update_pending to go high. */ 205 for (i = 0; i < rdev->usec_timeout; i++) { 206 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) 207 break; 208 DRM_UDELAY(1); 209 } 210 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n"); 211 212 /* Unlock the lock, so double-buffering can take place inside vblank */ 213 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 214 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 215 216 /* Return current update_pending status: */ 217 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING; 218 } 219 220 /* get temperature in millidegrees */ 221 int evergreen_get_temp(struct radeon_device *rdev) 222 { 223 u32 temp, toffset; 224 int actual_temp = 0; 225 226 if (rdev->family == CHIP_JUNIPER) { 227 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 228 TOFFSET_SHIFT; 229 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 230 TS0_ADC_DOUT_SHIFT; 231 232 if (toffset & 0x100) 233 actual_temp = temp / 2 - (0x200 - toffset); 234 else 235 actual_temp = temp / 2 + toffset; 236 237 actual_temp = actual_temp * 1000; 238 239 } else { 240 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 241 ASIC_T_SHIFT; 242 243 if (temp & 0x400) 244 actual_temp = -256; 245 else if (temp & 0x200) 246 actual_temp = 255; 247 else if (temp & 0x100) { 248 actual_temp = temp & 0x1ff; 249 actual_temp |= ~0x1ff; 250 } else 251 actual_temp = temp & 0xff; 252 253 actual_temp = (actual_temp * 1000) / 2; 254 } 255 256 return actual_temp; 257 } 258 259 int sumo_get_temp(struct radeon_device *rdev) 260 { 261 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 262 int actual_temp = temp - 49; 263 264 return actual_temp * 1000; 265 } 266 267 /** 268 * sumo_pm_init_profile - Initialize power profiles callback. 269 * 270 * @rdev: radeon_device pointer 271 * 272 * Initialize the power states used in profile mode 273 * (sumo, trinity, SI). 274 * Used for profile mode only. 275 */ 276 void sumo_pm_init_profile(struct radeon_device *rdev) 277 { 278 int idx; 279 280 /* default */ 281 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 282 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 283 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 284 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0; 285 286 /* low,mid sh/mh */ 287 if (rdev->flags & RADEON_IS_MOBILITY) 288 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 289 else 290 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 291 292 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 293 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 294 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 295 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 296 297 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 298 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 299 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 300 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 301 302 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 303 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 304 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 305 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0; 306 307 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 308 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 309 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 310 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0; 311 312 /* high sh/mh */ 313 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 314 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 315 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 316 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 317 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 318 rdev->pm.power_state[idx].num_clock_modes - 1; 319 320 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 321 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 322 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 323 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 324 rdev->pm.power_state[idx].num_clock_modes - 1; 325 } 326 327 /** 328 * btc_pm_init_profile - Initialize power profiles callback. 329 * 330 * @rdev: radeon_device pointer 331 * 332 * Initialize the power states used in profile mode 333 * (BTC, cayman). 334 * Used for profile mode only. 335 */ 336 void btc_pm_init_profile(struct radeon_device *rdev) 337 { 338 int idx; 339 340 /* default */ 341 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 342 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 343 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 344 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2; 345 /* starting with BTC, there is one state that is used for both 346 * MH and SH. Difference is that we always use the high clock index for 347 * mclk. 348 */ 349 if (rdev->flags & RADEON_IS_MOBILITY) 350 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 351 else 352 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 353 /* low sh */ 354 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 355 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 356 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 357 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 358 /* mid sh */ 359 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 360 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 361 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 362 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1; 363 /* high sh */ 364 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 365 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 366 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 367 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2; 368 /* low mh */ 369 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 370 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 371 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 372 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 373 /* mid mh */ 374 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 375 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 376 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 377 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1; 378 /* high mh */ 379 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 380 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 381 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 382 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2; 383 } 384 385 /** 386 * evergreen_pm_misc - set additional pm hw parameters callback. 387 * 388 * @rdev: radeon_device pointer 389 * 390 * Set non-clock parameters associated with a power state 391 * (voltage, etc.) (evergreen+). 392 */ 393 void evergreen_pm_misc(struct radeon_device *rdev) 394 { 395 int req_ps_idx = rdev->pm.requested_power_state_index; 396 int req_cm_idx = rdev->pm.requested_clock_mode_index; 397 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 398 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 399 400 if (voltage->type == VOLTAGE_SW) { 401 /* 0xff01 is a flag rather then an actual voltage */ 402 if (voltage->voltage == 0xff01) 403 return; 404 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 405 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 406 rdev->pm.current_vddc = voltage->voltage; 407 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 408 } 409 410 /* starting with BTC, there is one state that is used for both 411 * MH and SH. Difference is that we always use the high clock index for 412 * mclk and vddci. 413 */ 414 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) && 415 (rdev->family >= CHIP_BARTS) && 416 rdev->pm.active_crtc_count && 417 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) || 418 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX))) 419 voltage = &rdev->pm.power_state[req_ps_idx]. 420 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; 421 422 /* 0xff01 is a flag rather then an actual voltage */ 423 if (voltage->vddci == 0xff01) 424 return; 425 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 426 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 427 rdev->pm.current_vddci = voltage->vddci; 428 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 429 } 430 } 431 } 432 433 /** 434 * evergreen_pm_prepare - pre-power state change callback. 435 * 436 * @rdev: radeon_device pointer 437 * 438 * Prepare for a power state change (evergreen+). 439 */ 440 void evergreen_pm_prepare(struct radeon_device *rdev) 441 { 442 struct drm_device *ddev = rdev->ddev; 443 struct drm_crtc *crtc; 444 struct radeon_crtc *radeon_crtc; 445 u32 tmp; 446 447 /* disable any active CRTCs */ 448 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 449 radeon_crtc = to_radeon_crtc(crtc); 450 if (radeon_crtc->enabled) { 451 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 452 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 453 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 454 } 455 } 456 } 457 458 /** 459 * evergreen_pm_finish - post-power state change callback. 460 * 461 * @rdev: radeon_device pointer 462 * 463 * Clean up after a power state change (evergreen+). 464 */ 465 void evergreen_pm_finish(struct radeon_device *rdev) 466 { 467 struct drm_device *ddev = rdev->ddev; 468 struct drm_crtc *crtc; 469 struct radeon_crtc *radeon_crtc; 470 u32 tmp; 471 472 /* enable any active CRTCs */ 473 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 474 radeon_crtc = to_radeon_crtc(crtc); 475 if (radeon_crtc->enabled) { 476 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 477 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 478 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 479 } 480 } 481 } 482 483 /** 484 * evergreen_hpd_sense - hpd sense callback. 485 * 486 * @rdev: radeon_device pointer 487 * @hpd: hpd (hotplug detect) pin 488 * 489 * Checks if a digital monitor is connected (evergreen+). 490 * Returns true if connected, false if not connected. 491 */ 492 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 493 { 494 bool connected = false; 495 496 switch (hpd) { 497 case RADEON_HPD_1: 498 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE) 499 connected = true; 500 break; 501 case RADEON_HPD_2: 502 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE) 503 connected = true; 504 break; 505 case RADEON_HPD_3: 506 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE) 507 connected = true; 508 break; 509 case RADEON_HPD_4: 510 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE) 511 connected = true; 512 break; 513 case RADEON_HPD_5: 514 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE) 515 connected = true; 516 break; 517 case RADEON_HPD_6: 518 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) 519 connected = true; 520 break; 521 default: 522 break; 523 } 524 525 return connected; 526 } 527 528 /** 529 * evergreen_hpd_set_polarity - hpd set polarity callback. 530 * 531 * @rdev: radeon_device pointer 532 * @hpd: hpd (hotplug detect) pin 533 * 534 * Set the polarity of the hpd pin (evergreen+). 535 */ 536 void evergreen_hpd_set_polarity(struct radeon_device *rdev, 537 enum radeon_hpd_id hpd) 538 { 539 u32 tmp; 540 bool connected = evergreen_hpd_sense(rdev, hpd); 541 542 switch (hpd) { 543 case RADEON_HPD_1: 544 tmp = RREG32(DC_HPD1_INT_CONTROL); 545 if (connected) 546 tmp &= ~DC_HPDx_INT_POLARITY; 547 else 548 tmp |= DC_HPDx_INT_POLARITY; 549 WREG32(DC_HPD1_INT_CONTROL, tmp); 550 break; 551 case RADEON_HPD_2: 552 tmp = RREG32(DC_HPD2_INT_CONTROL); 553 if (connected) 554 tmp &= ~DC_HPDx_INT_POLARITY; 555 else 556 tmp |= DC_HPDx_INT_POLARITY; 557 WREG32(DC_HPD2_INT_CONTROL, tmp); 558 break; 559 case RADEON_HPD_3: 560 tmp = RREG32(DC_HPD3_INT_CONTROL); 561 if (connected) 562 tmp &= ~DC_HPDx_INT_POLARITY; 563 else 564 tmp |= DC_HPDx_INT_POLARITY; 565 WREG32(DC_HPD3_INT_CONTROL, tmp); 566 break; 567 case RADEON_HPD_4: 568 tmp = RREG32(DC_HPD4_INT_CONTROL); 569 if (connected) 570 tmp &= ~DC_HPDx_INT_POLARITY; 571 else 572 tmp |= DC_HPDx_INT_POLARITY; 573 WREG32(DC_HPD4_INT_CONTROL, tmp); 574 break; 575 case RADEON_HPD_5: 576 tmp = RREG32(DC_HPD5_INT_CONTROL); 577 if (connected) 578 tmp &= ~DC_HPDx_INT_POLARITY; 579 else 580 tmp |= DC_HPDx_INT_POLARITY; 581 WREG32(DC_HPD5_INT_CONTROL, tmp); 582 break; 583 case RADEON_HPD_6: 584 tmp = RREG32(DC_HPD6_INT_CONTROL); 585 if (connected) 586 tmp &= ~DC_HPDx_INT_POLARITY; 587 else 588 tmp |= DC_HPDx_INT_POLARITY; 589 WREG32(DC_HPD6_INT_CONTROL, tmp); 590 break; 591 default: 592 break; 593 } 594 } 595 596 /** 597 * evergreen_hpd_init - hpd setup callback. 598 * 599 * @rdev: radeon_device pointer 600 * 601 * Setup the hpd pins used by the card (evergreen+). 602 * Enable the pin, set the polarity, and enable the hpd interrupts. 603 */ 604 void evergreen_hpd_init(struct radeon_device *rdev) 605 { 606 struct drm_device *dev = rdev->ddev; 607 struct drm_connector *connector; 608 unsigned enabled = 0; 609 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 610 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 611 612 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 613 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 614 switch (radeon_connector->hpd.hpd) { 615 case RADEON_HPD_1: 616 WREG32(DC_HPD1_CONTROL, tmp); 617 break; 618 case RADEON_HPD_2: 619 WREG32(DC_HPD2_CONTROL, tmp); 620 break; 621 case RADEON_HPD_3: 622 WREG32(DC_HPD3_CONTROL, tmp); 623 break; 624 case RADEON_HPD_4: 625 WREG32(DC_HPD4_CONTROL, tmp); 626 break; 627 case RADEON_HPD_5: 628 WREG32(DC_HPD5_CONTROL, tmp); 629 break; 630 case RADEON_HPD_6: 631 WREG32(DC_HPD6_CONTROL, tmp); 632 break; 633 default: 634 break; 635 } 636 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd); 637 enabled |= 1 << radeon_connector->hpd.hpd; 638 } 639 radeon_irq_kms_enable_hpd(rdev, enabled); 640 } 641 642 /** 643 * evergreen_hpd_fini - hpd tear down callback. 644 * 645 * @rdev: radeon_device pointer 646 * 647 * Tear down the hpd pins used by the card (evergreen+). 648 * Disable the hpd interrupts. 649 */ 650 void evergreen_hpd_fini(struct radeon_device *rdev) 651 { 652 struct drm_device *dev = rdev->ddev; 653 struct drm_connector *connector; 654 unsigned disabled = 0; 655 656 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 657 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 658 switch (radeon_connector->hpd.hpd) { 659 case RADEON_HPD_1: 660 WREG32(DC_HPD1_CONTROL, 0); 661 break; 662 case RADEON_HPD_2: 663 WREG32(DC_HPD2_CONTROL, 0); 664 break; 665 case RADEON_HPD_3: 666 WREG32(DC_HPD3_CONTROL, 0); 667 break; 668 case RADEON_HPD_4: 669 WREG32(DC_HPD4_CONTROL, 0); 670 break; 671 case RADEON_HPD_5: 672 WREG32(DC_HPD5_CONTROL, 0); 673 break; 674 case RADEON_HPD_6: 675 WREG32(DC_HPD6_CONTROL, 0); 676 break; 677 default: 678 break; 679 } 680 disabled |= 1 << radeon_connector->hpd.hpd; 681 } 682 radeon_irq_kms_disable_hpd(rdev, disabled); 683 } 684 685 /* watermark setup */ 686 687 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 688 struct radeon_crtc *radeon_crtc, 689 struct drm_display_mode *mode, 690 struct drm_display_mode *other_mode) 691 { 692 u32 tmp; 693 /* 694 * Line Buffer Setup 695 * There are 3 line buffers, each one shared by 2 display controllers. 696 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 697 * the display controllers. The paritioning is done via one of four 698 * preset allocations specified in bits 2:0: 699 * first display controller 700 * 0 - first half of lb (3840 * 2) 701 * 1 - first 3/4 of lb (5760 * 2) 702 * 2 - whole lb (7680 * 2), other crtc must be disabled 703 * 3 - first 1/4 of lb (1920 * 2) 704 * second display controller 705 * 4 - second half of lb (3840 * 2) 706 * 5 - second 3/4 of lb (5760 * 2) 707 * 6 - whole lb (7680 * 2), other crtc must be disabled 708 * 7 - last 1/4 of lb (1920 * 2) 709 */ 710 /* this can get tricky if we have two large displays on a paired group 711 * of crtcs. Ideally for multiple large displays we'd assign them to 712 * non-linked crtcs for maximum line buffer allocation. 713 */ 714 if (radeon_crtc->base.enabled && mode) { 715 if (other_mode) 716 tmp = 0; /* 1/2 */ 717 else 718 tmp = 2; /* whole */ 719 } else 720 tmp = 0; 721 722 /* second controller of the pair uses second half of the lb */ 723 if (radeon_crtc->crtc_id % 2) 724 tmp += 4; 725 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 726 727 if (radeon_crtc->base.enabled && mode) { 728 switch (tmp) { 729 case 0: 730 case 4: 731 default: 732 if (ASIC_IS_DCE5(rdev)) 733 return 4096 * 2; 734 else 735 return 3840 * 2; 736 case 1: 737 case 5: 738 if (ASIC_IS_DCE5(rdev)) 739 return 6144 * 2; 740 else 741 return 5760 * 2; 742 case 2: 743 case 6: 744 if (ASIC_IS_DCE5(rdev)) 745 return 8192 * 2; 746 else 747 return 7680 * 2; 748 case 3: 749 case 7: 750 if (ASIC_IS_DCE5(rdev)) 751 return 2048 * 2; 752 else 753 return 1920 * 2; 754 } 755 } 756 757 /* controller not enabled, so no lb used */ 758 return 0; 759 } 760 761 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 762 { 763 u32 tmp = RREG32(MC_SHARED_CHMAP); 764 765 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 766 case 0: 767 default: 768 return 1; 769 case 1: 770 return 2; 771 case 2: 772 return 4; 773 case 3: 774 return 8; 775 } 776 } 777 778 struct evergreen_wm_params { 779 u32 dram_channels; /* number of dram channels */ 780 u32 yclk; /* bandwidth per dram data pin in kHz */ 781 u32 sclk; /* engine clock in kHz */ 782 u32 disp_clk; /* display clock in kHz */ 783 u32 src_width; /* viewport width */ 784 u32 active_time; /* active display time in ns */ 785 u32 blank_time; /* blank time in ns */ 786 bool interlaced; /* mode is interlaced */ 787 fixed20_12 vsc; /* vertical scale ratio */ 788 u32 num_heads; /* number of active crtcs */ 789 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 790 u32 lb_size; /* line buffer allocated to pipe */ 791 u32 vtaps; /* vertical scaler taps */ 792 }; 793 794 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 795 { 796 /* Calculate DRAM Bandwidth and the part allocated to display. */ 797 fixed20_12 dram_efficiency; /* 0.7 */ 798 fixed20_12 yclk, dram_channels, bandwidth; 799 fixed20_12 a; 800 801 a.full = dfixed_const(1000); 802 yclk.full = dfixed_const(wm->yclk); 803 yclk.full = dfixed_div(yclk, a); 804 dram_channels.full = dfixed_const(wm->dram_channels * 4); 805 a.full = dfixed_const(10); 806 dram_efficiency.full = dfixed_const(7); 807 dram_efficiency.full = dfixed_div(dram_efficiency, a); 808 bandwidth.full = dfixed_mul(dram_channels, yclk); 809 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 810 811 return dfixed_trunc(bandwidth); 812 } 813 814 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 815 { 816 /* Calculate DRAM Bandwidth and the part allocated to display. */ 817 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 818 fixed20_12 yclk, dram_channels, bandwidth; 819 fixed20_12 a; 820 821 a.full = dfixed_const(1000); 822 yclk.full = dfixed_const(wm->yclk); 823 yclk.full = dfixed_div(yclk, a); 824 dram_channels.full = dfixed_const(wm->dram_channels * 4); 825 a.full = dfixed_const(10); 826 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 827 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 828 bandwidth.full = dfixed_mul(dram_channels, yclk); 829 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 830 831 return dfixed_trunc(bandwidth); 832 } 833 834 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 835 { 836 /* Calculate the display Data return Bandwidth */ 837 fixed20_12 return_efficiency; /* 0.8 */ 838 fixed20_12 sclk, bandwidth; 839 fixed20_12 a; 840 841 a.full = dfixed_const(1000); 842 sclk.full = dfixed_const(wm->sclk); 843 sclk.full = dfixed_div(sclk, a); 844 a.full = dfixed_const(10); 845 return_efficiency.full = dfixed_const(8); 846 return_efficiency.full = dfixed_div(return_efficiency, a); 847 a.full = dfixed_const(32); 848 bandwidth.full = dfixed_mul(a, sclk); 849 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 850 851 return dfixed_trunc(bandwidth); 852 } 853 854 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 855 { 856 /* Calculate the DMIF Request Bandwidth */ 857 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 858 fixed20_12 disp_clk, bandwidth; 859 fixed20_12 a; 860 861 a.full = dfixed_const(1000); 862 disp_clk.full = dfixed_const(wm->disp_clk); 863 disp_clk.full = dfixed_div(disp_clk, a); 864 a.full = dfixed_const(10); 865 disp_clk_request_efficiency.full = dfixed_const(8); 866 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 867 a.full = dfixed_const(32); 868 bandwidth.full = dfixed_mul(a, disp_clk); 869 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 870 871 return dfixed_trunc(bandwidth); 872 } 873 874 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 875 { 876 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 877 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 878 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 879 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 880 881 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 882 } 883 884 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 885 { 886 /* Calculate the display mode Average Bandwidth 887 * DisplayMode should contain the source and destination dimensions, 888 * timing, etc. 889 */ 890 fixed20_12 bpp; 891 fixed20_12 line_time; 892 fixed20_12 src_width; 893 fixed20_12 bandwidth; 894 fixed20_12 a; 895 896 a.full = dfixed_const(1000); 897 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 898 line_time.full = dfixed_div(line_time, a); 899 bpp.full = dfixed_const(wm->bytes_per_pixel); 900 src_width.full = dfixed_const(wm->src_width); 901 bandwidth.full = dfixed_mul(src_width, bpp); 902 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 903 bandwidth.full = dfixed_div(bandwidth, line_time); 904 905 return dfixed_trunc(bandwidth); 906 } 907 908 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 909 { 910 /* First calcualte the latency in ns */ 911 u32 mc_latency = 2000; /* 2000 ns. */ 912 u32 available_bandwidth = evergreen_available_bandwidth(wm); 913 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 914 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 915 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 916 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 917 (wm->num_heads * cursor_line_pair_return_time); 918 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 919 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 920 fixed20_12 a, b, c; 921 922 if (wm->num_heads == 0) 923 return 0; 924 925 a.full = dfixed_const(2); 926 b.full = dfixed_const(1); 927 if ((wm->vsc.full > a.full) || 928 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 929 (wm->vtaps >= 5) || 930 ((wm->vsc.full >= a.full) && wm->interlaced)) 931 max_src_lines_per_dst_line = 4; 932 else 933 max_src_lines_per_dst_line = 2; 934 935 a.full = dfixed_const(available_bandwidth); 936 b.full = dfixed_const(wm->num_heads); 937 a.full = dfixed_div(a, b); 938 939 b.full = dfixed_const(1000); 940 c.full = dfixed_const(wm->disp_clk); 941 b.full = dfixed_div(c, b); 942 c.full = dfixed_const(wm->bytes_per_pixel); 943 b.full = dfixed_mul(b, c); 944 945 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b)); 946 947 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 948 b.full = dfixed_const(1000); 949 c.full = dfixed_const(lb_fill_bw); 950 b.full = dfixed_div(c, b); 951 a.full = dfixed_div(a, b); 952 line_fill_time = dfixed_trunc(a); 953 954 if (line_fill_time < wm->active_time) 955 return latency; 956 else 957 return latency + (line_fill_time - wm->active_time); 958 959 } 960 961 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 962 { 963 if (evergreen_average_bandwidth(wm) <= 964 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 965 return true; 966 else 967 return false; 968 }; 969 970 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 971 { 972 if (evergreen_average_bandwidth(wm) <= 973 (evergreen_available_bandwidth(wm) / wm->num_heads)) 974 return true; 975 else 976 return false; 977 }; 978 979 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 980 { 981 u32 lb_partitions = wm->lb_size / wm->src_width; 982 u32 line_time = wm->active_time + wm->blank_time; 983 u32 latency_tolerant_lines; 984 u32 latency_hiding; 985 fixed20_12 a; 986 987 a.full = dfixed_const(1); 988 if (wm->vsc.full > a.full) 989 latency_tolerant_lines = 1; 990 else { 991 if (lb_partitions <= (wm->vtaps + 1)) 992 latency_tolerant_lines = 1; 993 else 994 latency_tolerant_lines = 2; 995 } 996 997 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 998 999 if (evergreen_latency_watermark(wm) <= latency_hiding) 1000 return true; 1001 else 1002 return false; 1003 } 1004 1005 static void evergreen_program_watermarks(struct radeon_device *rdev, 1006 struct radeon_crtc *radeon_crtc, 1007 u32 lb_size, u32 num_heads) 1008 { 1009 struct drm_display_mode *mode = &radeon_crtc->base.mode; 1010 struct evergreen_wm_params wm; 1011 u32 pixel_period; 1012 u32 line_time = 0; 1013 u32 latency_watermark_a = 0, latency_watermark_b = 0; 1014 u32 priority_a_mark = 0, priority_b_mark = 0; 1015 u32 priority_a_cnt = PRIORITY_OFF; 1016 u32 priority_b_cnt = PRIORITY_OFF; 1017 u32 pipe_offset = radeon_crtc->crtc_id * 16; 1018 u32 tmp, arb_control3; 1019 fixed20_12 a, b, c; 1020 1021 if (radeon_crtc->base.enabled && num_heads && mode) { 1022 pixel_period = 1000000 / (u32)mode->clock; 1023 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 1024 priority_a_cnt = 0; 1025 priority_b_cnt = 0; 1026 1027 wm.yclk = rdev->pm.current_mclk * 10; 1028 wm.sclk = rdev->pm.current_sclk * 10; 1029 wm.disp_clk = mode->clock; 1030 wm.src_width = mode->crtc_hdisplay; 1031 wm.active_time = mode->crtc_hdisplay * pixel_period; 1032 wm.blank_time = line_time - wm.active_time; 1033 wm.interlaced = false; 1034 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 1035 wm.interlaced = true; 1036 wm.vsc = radeon_crtc->vsc; 1037 wm.vtaps = 1; 1038 if (radeon_crtc->rmx_type != RMX_OFF) 1039 wm.vtaps = 2; 1040 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */ 1041 wm.lb_size = lb_size; 1042 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev); 1043 wm.num_heads = num_heads; 1044 1045 /* set for high clocks */ 1046 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535); 1047 /* set for low clocks */ 1048 /* wm.yclk = low clk; wm.sclk = low clk */ 1049 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535); 1050 1051 /* possibly force display priority to high */ 1052 /* should really do this at mode validation time... */ 1053 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) || 1054 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) || 1055 !evergreen_check_latency_hiding(&wm) || 1056 (rdev->disp_priority == 2)) { 1057 DRM_DEBUG_KMS("force priority to high\n"); 1058 priority_a_cnt |= PRIORITY_ALWAYS_ON; 1059 priority_b_cnt |= PRIORITY_ALWAYS_ON; 1060 } 1061 1062 a.full = dfixed_const(1000); 1063 b.full = dfixed_const(mode->clock); 1064 b.full = dfixed_div(b, a); 1065 c.full = dfixed_const(latency_watermark_a); 1066 c.full = dfixed_mul(c, b); 1067 c.full = dfixed_mul(c, radeon_crtc->hsc); 1068 c.full = dfixed_div(c, a); 1069 a.full = dfixed_const(16); 1070 c.full = dfixed_div(c, a); 1071 priority_a_mark = dfixed_trunc(c); 1072 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 1073 1074 a.full = dfixed_const(1000); 1075 b.full = dfixed_const(mode->clock); 1076 b.full = dfixed_div(b, a); 1077 c.full = dfixed_const(latency_watermark_b); 1078 c.full = dfixed_mul(c, b); 1079 c.full = dfixed_mul(c, radeon_crtc->hsc); 1080 c.full = dfixed_div(c, a); 1081 a.full = dfixed_const(16); 1082 c.full = dfixed_div(c, a); 1083 priority_b_mark = dfixed_trunc(c); 1084 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 1085 } 1086 1087 /* select wm A */ 1088 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 1089 tmp = arb_control3; 1090 tmp &= ~LATENCY_WATERMARK_MASK(3); 1091 tmp |= LATENCY_WATERMARK_MASK(1); 1092 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 1093 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 1094 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 1095 LATENCY_HIGH_WATERMARK(line_time))); 1096 /* select wm B */ 1097 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 1098 tmp &= ~LATENCY_WATERMARK_MASK(3); 1099 tmp |= LATENCY_WATERMARK_MASK(2); 1100 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 1101 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 1102 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 1103 LATENCY_HIGH_WATERMARK(line_time))); 1104 /* restore original selection */ 1105 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 1106 1107 /* write the priority marks */ 1108 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 1109 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 1110 1111 } 1112 1113 /** 1114 * evergreen_bandwidth_update - update display watermarks callback. 1115 * 1116 * @rdev: radeon_device pointer 1117 * 1118 * Update the display watermarks based on the requested mode(s) 1119 * (evergreen+). 1120 */ 1121 void evergreen_bandwidth_update(struct radeon_device *rdev) 1122 { 1123 struct drm_display_mode *mode0 = NULL; 1124 struct drm_display_mode *mode1 = NULL; 1125 u32 num_heads = 0, lb_size; 1126 int i; 1127 1128 radeon_update_display_priority(rdev); 1129 1130 for (i = 0; i < rdev->num_crtc; i++) { 1131 if (rdev->mode_info.crtcs[i]->base.enabled) 1132 num_heads++; 1133 } 1134 for (i = 0; i < rdev->num_crtc; i += 2) { 1135 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 1136 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 1137 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 1138 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 1139 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 1140 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 1141 } 1142 } 1143 1144 /** 1145 * evergreen_mc_wait_for_idle - wait for MC idle callback. 1146 * 1147 * @rdev: radeon_device pointer 1148 * 1149 * Wait for the MC (memory controller) to be idle. 1150 * (evergreen+). 1151 * Returns 0 if the MC is idle, -1 if not. 1152 */ 1153 int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 1154 { 1155 unsigned i; 1156 u32 tmp; 1157 1158 for (i = 0; i < rdev->usec_timeout; i++) { 1159 /* read MC_STATUS */ 1160 tmp = RREG32(SRBM_STATUS) & 0x1F00; 1161 if (!tmp) 1162 return 0; 1163 DRM_UDELAY(1); 1164 } 1165 return -1; 1166 } 1167 1168 /* 1169 * GART 1170 */ 1171 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 1172 { 1173 unsigned i; 1174 u32 tmp; 1175 1176 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 1177 1178 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 1179 for (i = 0; i < rdev->usec_timeout; i++) { 1180 /* read MC_STATUS */ 1181 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 1182 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 1183 if (tmp == 2) { 1184 DRM_ERROR("[drm] r600 flush TLB failed\n"); 1185 return; 1186 } 1187 if (tmp) { 1188 return; 1189 } 1190 DRM_UDELAY(1); 1191 } 1192 } 1193 1194 static int evergreen_pcie_gart_enable(struct radeon_device *rdev) 1195 { 1196 u32 tmp; 1197 int r; 1198 1199 if (rdev->gart.robj == NULL) { 1200 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 1201 return -EINVAL; 1202 } 1203 r = radeon_gart_table_vram_pin(rdev); 1204 if (r) 1205 return r; 1206 radeon_gart_restore(rdev); 1207 /* Setup L2 cache */ 1208 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 1209 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 1210 EFFECTIVE_L2_QUEUE_SIZE(7)); 1211 WREG32(VM_L2_CNTL2, 0); 1212 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 1213 /* Setup TLB control */ 1214 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 1215 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 1216 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 1217 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 1218 if (rdev->flags & RADEON_IS_IGP) { 1219 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 1220 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 1221 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 1222 } else { 1223 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 1224 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 1225 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 1226 if ((rdev->family == CHIP_JUNIPER) || 1227 (rdev->family == CHIP_CYPRESS) || 1228 (rdev->family == CHIP_HEMLOCK) || 1229 (rdev->family == CHIP_BARTS)) 1230 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp); 1231 } 1232 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 1233 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 1234 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 1235 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 1236 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 1237 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 1238 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 1239 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 1240 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 1241 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 1242 (u32)(rdev->dummy_page.addr >> 12)); 1243 WREG32(VM_CONTEXT1_CNTL, 0); 1244 1245 evergreen_pcie_gart_tlb_flush(rdev); 1246 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1247 (unsigned)(rdev->mc.gtt_size >> 20), 1248 (unsigned long long)rdev->gart.table_addr); 1249 rdev->gart.ready = true; 1250 return 0; 1251 } 1252 1253 static void evergreen_pcie_gart_disable(struct radeon_device *rdev) 1254 { 1255 u32 tmp; 1256 1257 /* Disable all tables */ 1258 WREG32(VM_CONTEXT0_CNTL, 0); 1259 WREG32(VM_CONTEXT1_CNTL, 0); 1260 1261 /* Setup L2 cache */ 1262 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 1263 EFFECTIVE_L2_QUEUE_SIZE(7)); 1264 WREG32(VM_L2_CNTL2, 0); 1265 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 1266 /* Setup TLB control */ 1267 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 1268 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 1269 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 1270 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 1271 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 1272 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 1273 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 1274 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 1275 radeon_gart_table_vram_unpin(rdev); 1276 } 1277 1278 static void evergreen_pcie_gart_fini(struct radeon_device *rdev) 1279 { 1280 evergreen_pcie_gart_disable(rdev); 1281 radeon_gart_table_vram_free(rdev); 1282 radeon_gart_fini(rdev); 1283 } 1284 1285 1286 static void evergreen_agp_enable(struct radeon_device *rdev) 1287 { 1288 u32 tmp; 1289 1290 /* Setup L2 cache */ 1291 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 1292 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 1293 EFFECTIVE_L2_QUEUE_SIZE(7)); 1294 WREG32(VM_L2_CNTL2, 0); 1295 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 1296 /* Setup TLB control */ 1297 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 1298 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 1299 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 1300 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 1301 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 1302 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 1303 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 1304 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 1305 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 1306 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 1307 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 1308 WREG32(VM_CONTEXT0_CNTL, 0); 1309 WREG32(VM_CONTEXT1_CNTL, 0); 1310 } 1311 1312 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 1313 { 1314 u32 crtc_enabled, tmp, frame_count, blackout; 1315 int i, j; 1316 1317 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 1318 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 1319 1320 /* disable VGA render */ 1321 WREG32(VGA_RENDER_CONTROL, 0); 1322 /* blank the display controllers */ 1323 for (i = 0; i < rdev->num_crtc; i++) { 1324 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; 1325 if (crtc_enabled) { 1326 save->crtc_enabled[i] = true; 1327 if (ASIC_IS_DCE6(rdev)) { 1328 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 1329 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) { 1330 radeon_wait_for_vblank(rdev, i); 1331 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 1332 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 1333 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 1334 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 1335 } 1336 } else { 1337 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 1338 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) { 1339 radeon_wait_for_vblank(rdev, i); 1340 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1341 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 1342 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 1343 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 1344 } 1345 } 1346 /* wait for the next frame */ 1347 frame_count = radeon_get_vblank_counter(rdev, i); 1348 for (j = 0; j < rdev->usec_timeout; j++) { 1349 if (radeon_get_vblank_counter(rdev, i) != frame_count) 1350 break; 1351 DRM_UDELAY(1); 1352 } 1353 } else { 1354 save->crtc_enabled[i] = false; 1355 } 1356 } 1357 1358 radeon_mc_wait_for_idle(rdev); 1359 1360 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1361 if ((blackout & BLACKOUT_MODE_MASK) != 1) { 1362 /* Block CPU access */ 1363 WREG32(BIF_FB_EN, 0); 1364 /* blackout the MC */ 1365 blackout &= ~BLACKOUT_MODE_MASK; 1366 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1367 } 1368 /* wait for the MC to settle */ 1369 DRM_UDELAY(100); 1370 } 1371 1372 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 1373 { 1374 u32 tmp, frame_count; 1375 int i, j; 1376 1377 /* update crtc base addresses */ 1378 for (i = 0; i < rdev->num_crtc; i++) { 1379 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 1380 upper_32_bits(rdev->mc.vram_start)); 1381 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 1382 upper_32_bits(rdev->mc.vram_start)); 1383 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], 1384 (u32)rdev->mc.vram_start); 1385 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], 1386 (u32)rdev->mc.vram_start); 1387 } 1388 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 1389 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 1390 1391 /* unblackout the MC */ 1392 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL); 1393 tmp &= ~BLACKOUT_MODE_MASK; 1394 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp); 1395 /* allow CPU access */ 1396 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 1397 1398 for (i = 0; i < rdev->num_crtc; i++) { 1399 if (save->crtc_enabled[i]) { 1400 if (ASIC_IS_DCE6(rdev)) { 1401 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 1402 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 1403 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 1404 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 1405 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 1406 } else { 1407 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 1408 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1409 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 1410 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 1411 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 1412 } 1413 /* wait for the next frame */ 1414 frame_count = radeon_get_vblank_counter(rdev, i); 1415 for (j = 0; j < rdev->usec_timeout; j++) { 1416 if (radeon_get_vblank_counter(rdev, i) != frame_count) 1417 break; 1418 DRM_UDELAY(1); 1419 } 1420 } 1421 } 1422 /* Unlock vga access */ 1423 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 1424 DRM_MDELAY(1); 1425 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 1426 } 1427 1428 void evergreen_mc_program(struct radeon_device *rdev) 1429 { 1430 struct evergreen_mc_save save; 1431 u32 tmp; 1432 int i, j; 1433 1434 /* Initialize HDP */ 1435 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 1436 WREG32((0x2c14 + j), 0x00000000); 1437 WREG32((0x2c18 + j), 0x00000000); 1438 WREG32((0x2c1c + j), 0x00000000); 1439 WREG32((0x2c20 + j), 0x00000000); 1440 WREG32((0x2c24 + j), 0x00000000); 1441 } 1442 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 1443 1444 evergreen_mc_stop(rdev, &save); 1445 if (evergreen_mc_wait_for_idle(rdev)) { 1446 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 1447 } 1448 /* Lockout access through VGA aperture*/ 1449 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 1450 /* Update configuration */ 1451 if (rdev->flags & RADEON_IS_AGP) { 1452 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 1453 /* VRAM before AGP */ 1454 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 1455 rdev->mc.vram_start >> 12); 1456 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 1457 rdev->mc.gtt_end >> 12); 1458 } else { 1459 /* VRAM after AGP */ 1460 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 1461 rdev->mc.gtt_start >> 12); 1462 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 1463 rdev->mc.vram_end >> 12); 1464 } 1465 } else { 1466 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 1467 rdev->mc.vram_start >> 12); 1468 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 1469 rdev->mc.vram_end >> 12); 1470 } 1471 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12); 1472 /* llano/ontario only */ 1473 if ((rdev->family == CHIP_PALM) || 1474 (rdev->family == CHIP_SUMO) || 1475 (rdev->family == CHIP_SUMO2)) { 1476 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 1477 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 1478 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 1479 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 1480 } 1481 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 1482 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 1483 WREG32(MC_VM_FB_LOCATION, tmp); 1484 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 1485 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 1486 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 1487 if (rdev->flags & RADEON_IS_AGP) { 1488 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 1489 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 1490 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 1491 } else { 1492 WREG32(MC_VM_AGP_BASE, 0); 1493 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 1494 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 1495 } 1496 if (evergreen_mc_wait_for_idle(rdev)) { 1497 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 1498 } 1499 evergreen_mc_resume(rdev, &save); 1500 /* we need to own VRAM, so turn off the VGA renderer here 1501 * to stop it overwriting our objects */ 1502 rv515_vga_render_disable(rdev); 1503 } 1504 1505 /* 1506 * CP. 1507 */ 1508 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 1509 { 1510 struct radeon_ring *ring = &rdev->ring[ib->ring]; 1511 u32 next_rptr; 1512 1513 /* set to DX10/11 mode */ 1514 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 1515 radeon_ring_write(ring, 1); 1516 1517 if (ring->rptr_save_reg) { 1518 next_rptr = ring->wptr + 3 + 4; 1519 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 1520 radeon_ring_write(ring, ((ring->rptr_save_reg - 1521 PACKET3_SET_CONFIG_REG_START) >> 2)); 1522 radeon_ring_write(ring, next_rptr); 1523 } else if (rdev->wb.enabled) { 1524 next_rptr = ring->wptr + 5 + 4; 1525 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3)); 1526 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 1527 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18)); 1528 radeon_ring_write(ring, next_rptr); 1529 radeon_ring_write(ring, 0); 1530 } 1531 1532 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 1533 radeon_ring_write(ring, 1534 #ifdef __BIG_ENDIAN 1535 (2 << 0) | 1536 #endif 1537 (ib->gpu_addr & 0xFFFFFFFC)); 1538 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 1539 radeon_ring_write(ring, ib->length_dw); 1540 } 1541 1542 1543 static int evergreen_cp_load_microcode(struct radeon_device *rdev) 1544 { 1545 const __be32 *fw_data; 1546 int i; 1547 1548 if (!rdev->me_fw || !rdev->pfp_fw) 1549 return -EINVAL; 1550 1551 r700_cp_stop(rdev); 1552 WREG32(CP_RB_CNTL, 1553 #ifdef __BIG_ENDIAN 1554 BUF_SWAP_32BIT | 1555 #endif 1556 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 1557 1558 fw_data = (const __be32 *)rdev->pfp_fw->data; 1559 WREG32(CP_PFP_UCODE_ADDR, 0); 1560 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 1561 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 1562 WREG32(CP_PFP_UCODE_ADDR, 0); 1563 1564 fw_data = (const __be32 *)rdev->me_fw->data; 1565 WREG32(CP_ME_RAM_WADDR, 0); 1566 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 1567 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 1568 1569 WREG32(CP_PFP_UCODE_ADDR, 0); 1570 WREG32(CP_ME_RAM_WADDR, 0); 1571 WREG32(CP_ME_RAM_RADDR, 0); 1572 return 0; 1573 } 1574 1575 static int evergreen_cp_start(struct radeon_device *rdev) 1576 { 1577 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 1578 int r, i; 1579 uint32_t cp_me; 1580 1581 r = radeon_ring_lock(rdev, ring, 7); 1582 if (r) { 1583 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1584 return r; 1585 } 1586 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 1587 radeon_ring_write(ring, 0x1); 1588 radeon_ring_write(ring, 0x0); 1589 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1); 1590 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 1591 radeon_ring_write(ring, 0); 1592 radeon_ring_write(ring, 0); 1593 radeon_ring_unlock_commit(rdev, ring); 1594 1595 cp_me = 0xff; 1596 WREG32(CP_ME_CNTL, cp_me); 1597 1598 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19); 1599 if (r) { 1600 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1601 return r; 1602 } 1603 1604 /* setup clear context state */ 1605 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1606 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1607 1608 for (i = 0; i < evergreen_default_size; i++) 1609 radeon_ring_write(ring, evergreen_default_state[i]); 1610 1611 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1612 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 1613 1614 /* set clear context state */ 1615 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 1616 radeon_ring_write(ring, 0); 1617 1618 /* SQ_VTX_BASE_VTX_LOC */ 1619 radeon_ring_write(ring, 0xc0026f00); 1620 radeon_ring_write(ring, 0x00000000); 1621 radeon_ring_write(ring, 0x00000000); 1622 radeon_ring_write(ring, 0x00000000); 1623 1624 /* Clear consts */ 1625 radeon_ring_write(ring, 0xc0036f00); 1626 radeon_ring_write(ring, 0x00000bc4); 1627 radeon_ring_write(ring, 0xffffffff); 1628 radeon_ring_write(ring, 0xffffffff); 1629 radeon_ring_write(ring, 0xffffffff); 1630 1631 radeon_ring_write(ring, 0xc0026900); 1632 radeon_ring_write(ring, 0x00000316); 1633 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 1634 radeon_ring_write(ring, 0x00000010); /* */ 1635 1636 radeon_ring_unlock_commit(rdev, ring); 1637 1638 return 0; 1639 } 1640 1641 static int evergreen_cp_resume(struct radeon_device *rdev) 1642 { 1643 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 1644 u32 tmp; 1645 u32 rb_bufsz; 1646 int r; 1647 1648 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 1649 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 1650 SOFT_RESET_PA | 1651 SOFT_RESET_SH | 1652 SOFT_RESET_VGT | 1653 SOFT_RESET_SPI | 1654 SOFT_RESET_SX)); 1655 RREG32(GRBM_SOFT_RESET); 1656 DRM_MDELAY(15); 1657 WREG32(GRBM_SOFT_RESET, 0); 1658 RREG32(GRBM_SOFT_RESET); 1659 1660 /* Set ring buffer size */ 1661 rb_bufsz = drm_order(ring->ring_size / 8); 1662 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 1663 #ifdef __BIG_ENDIAN 1664 tmp |= BUF_SWAP_32BIT; 1665 #endif 1666 WREG32(CP_RB_CNTL, tmp); 1667 WREG32(CP_SEM_WAIT_TIMER, 0x0); 1668 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 1669 1670 /* Set the write pointer delay */ 1671 WREG32(CP_RB_WPTR_DELAY, 0); 1672 1673 /* Initialize the ring buffer's read and write pointers */ 1674 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 1675 WREG32(CP_RB_RPTR_WR, 0); 1676 ring->wptr = 0; 1677 WREG32(CP_RB_WPTR, ring->wptr); 1678 1679 /* set the wb address whether it's enabled or not */ 1680 WREG32(CP_RB_RPTR_ADDR, 1681 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 1682 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 1683 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 1684 1685 if (rdev->wb.enabled) 1686 WREG32(SCRATCH_UMSK, 0xff); 1687 else { 1688 tmp |= RB_NO_UPDATE; 1689 WREG32(SCRATCH_UMSK, 0); 1690 } 1691 1692 DRM_MDELAY(1); 1693 WREG32(CP_RB_CNTL, tmp); 1694 1695 WREG32(CP_RB_BASE, ring->gpu_addr >> 8); 1696 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 1697 1698 ring->rptr = RREG32(CP_RB_RPTR); 1699 1700 evergreen_cp_start(rdev); 1701 ring->ready = true; 1702 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); 1703 if (r) { 1704 ring->ready = false; 1705 return r; 1706 } 1707 return 0; 1708 } 1709 1710 /* 1711 * Core functions 1712 */ 1713 static void evergreen_gpu_init(struct radeon_device *rdev) 1714 { 1715 u32 gb_addr_config; 1716 u32 mc_shared_chmap, mc_arb_ramcfg; 1717 u32 sx_debug_1; 1718 u32 smx_dc_ctl0; 1719 u32 sq_config; 1720 u32 sq_lds_resource_mgmt; 1721 u32 sq_gpr_resource_mgmt_1; 1722 u32 sq_gpr_resource_mgmt_2; 1723 u32 sq_gpr_resource_mgmt_3; 1724 u32 sq_thread_resource_mgmt; 1725 u32 sq_thread_resource_mgmt_2; 1726 u32 sq_stack_resource_mgmt_1; 1727 u32 sq_stack_resource_mgmt_2; 1728 u32 sq_stack_resource_mgmt_3; 1729 u32 vgt_cache_invalidation; 1730 u32 hdp_host_path_cntl, tmp; 1731 u32 disabled_rb_mask; 1732 int i, j, num_shader_engines, ps_thread_count; 1733 1734 switch (rdev->family) { 1735 case CHIP_CYPRESS: 1736 case CHIP_HEMLOCK: 1737 rdev->config.evergreen.num_ses = 2; 1738 rdev->config.evergreen.max_pipes = 4; 1739 rdev->config.evergreen.max_tile_pipes = 8; 1740 rdev->config.evergreen.max_simds = 10; 1741 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 1742 rdev->config.evergreen.max_gprs = 256; 1743 rdev->config.evergreen.max_threads = 248; 1744 rdev->config.evergreen.max_gs_threads = 32; 1745 rdev->config.evergreen.max_stack_entries = 512; 1746 rdev->config.evergreen.sx_num_of_sets = 4; 1747 rdev->config.evergreen.sx_max_export_size = 256; 1748 rdev->config.evergreen.sx_max_export_pos_size = 64; 1749 rdev->config.evergreen.sx_max_export_smx_size = 192; 1750 rdev->config.evergreen.max_hw_contexts = 8; 1751 rdev->config.evergreen.sq_num_cf_insts = 2; 1752 1753 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1754 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1755 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1756 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN; 1757 break; 1758 case CHIP_JUNIPER: 1759 rdev->config.evergreen.num_ses = 1; 1760 rdev->config.evergreen.max_pipes = 4; 1761 rdev->config.evergreen.max_tile_pipes = 4; 1762 rdev->config.evergreen.max_simds = 10; 1763 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 1764 rdev->config.evergreen.max_gprs = 256; 1765 rdev->config.evergreen.max_threads = 248; 1766 rdev->config.evergreen.max_gs_threads = 32; 1767 rdev->config.evergreen.max_stack_entries = 512; 1768 rdev->config.evergreen.sx_num_of_sets = 4; 1769 rdev->config.evergreen.sx_max_export_size = 256; 1770 rdev->config.evergreen.sx_max_export_pos_size = 64; 1771 rdev->config.evergreen.sx_max_export_smx_size = 192; 1772 rdev->config.evergreen.max_hw_contexts = 8; 1773 rdev->config.evergreen.sq_num_cf_insts = 2; 1774 1775 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1776 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1777 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1778 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN; 1779 break; 1780 case CHIP_REDWOOD: 1781 rdev->config.evergreen.num_ses = 1; 1782 rdev->config.evergreen.max_pipes = 4; 1783 rdev->config.evergreen.max_tile_pipes = 4; 1784 rdev->config.evergreen.max_simds = 5; 1785 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 1786 rdev->config.evergreen.max_gprs = 256; 1787 rdev->config.evergreen.max_threads = 248; 1788 rdev->config.evergreen.max_gs_threads = 32; 1789 rdev->config.evergreen.max_stack_entries = 256; 1790 rdev->config.evergreen.sx_num_of_sets = 4; 1791 rdev->config.evergreen.sx_max_export_size = 256; 1792 rdev->config.evergreen.sx_max_export_pos_size = 64; 1793 rdev->config.evergreen.sx_max_export_smx_size = 192; 1794 rdev->config.evergreen.max_hw_contexts = 8; 1795 rdev->config.evergreen.sq_num_cf_insts = 2; 1796 1797 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1798 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1799 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1800 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN; 1801 break; 1802 case CHIP_CEDAR: 1803 default: 1804 rdev->config.evergreen.num_ses = 1; 1805 rdev->config.evergreen.max_pipes = 2; 1806 rdev->config.evergreen.max_tile_pipes = 2; 1807 rdev->config.evergreen.max_simds = 2; 1808 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1809 rdev->config.evergreen.max_gprs = 256; 1810 rdev->config.evergreen.max_threads = 192; 1811 rdev->config.evergreen.max_gs_threads = 16; 1812 rdev->config.evergreen.max_stack_entries = 256; 1813 rdev->config.evergreen.sx_num_of_sets = 4; 1814 rdev->config.evergreen.sx_max_export_size = 128; 1815 rdev->config.evergreen.sx_max_export_pos_size = 32; 1816 rdev->config.evergreen.sx_max_export_smx_size = 96; 1817 rdev->config.evergreen.max_hw_contexts = 4; 1818 rdev->config.evergreen.sq_num_cf_insts = 1; 1819 1820 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1821 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1822 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1823 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 1824 break; 1825 case CHIP_PALM: 1826 rdev->config.evergreen.num_ses = 1; 1827 rdev->config.evergreen.max_pipes = 2; 1828 rdev->config.evergreen.max_tile_pipes = 2; 1829 rdev->config.evergreen.max_simds = 2; 1830 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1831 rdev->config.evergreen.max_gprs = 256; 1832 rdev->config.evergreen.max_threads = 192; 1833 rdev->config.evergreen.max_gs_threads = 16; 1834 rdev->config.evergreen.max_stack_entries = 256; 1835 rdev->config.evergreen.sx_num_of_sets = 4; 1836 rdev->config.evergreen.sx_max_export_size = 128; 1837 rdev->config.evergreen.sx_max_export_pos_size = 32; 1838 rdev->config.evergreen.sx_max_export_smx_size = 96; 1839 rdev->config.evergreen.max_hw_contexts = 4; 1840 rdev->config.evergreen.sq_num_cf_insts = 1; 1841 1842 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1843 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1844 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1845 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 1846 break; 1847 case CHIP_SUMO: 1848 rdev->config.evergreen.num_ses = 1; 1849 rdev->config.evergreen.max_pipes = 4; 1850 rdev->config.evergreen.max_tile_pipes = 4; 1851 if (rdev->ddev->pci_device == 0x9648) 1852 rdev->config.evergreen.max_simds = 3; 1853 else if ((rdev->ddev->pci_device == 0x9647) || 1854 (rdev->ddev->pci_device == 0x964a)) 1855 rdev->config.evergreen.max_simds = 4; 1856 else 1857 rdev->config.evergreen.max_simds = 5; 1858 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 1859 rdev->config.evergreen.max_gprs = 256; 1860 rdev->config.evergreen.max_threads = 248; 1861 rdev->config.evergreen.max_gs_threads = 32; 1862 rdev->config.evergreen.max_stack_entries = 256; 1863 rdev->config.evergreen.sx_num_of_sets = 4; 1864 rdev->config.evergreen.sx_max_export_size = 256; 1865 rdev->config.evergreen.sx_max_export_pos_size = 64; 1866 rdev->config.evergreen.sx_max_export_smx_size = 192; 1867 rdev->config.evergreen.max_hw_contexts = 8; 1868 rdev->config.evergreen.sq_num_cf_insts = 2; 1869 1870 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1871 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1872 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1873 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN; 1874 break; 1875 case CHIP_SUMO2: 1876 rdev->config.evergreen.num_ses = 1; 1877 rdev->config.evergreen.max_pipes = 4; 1878 rdev->config.evergreen.max_tile_pipes = 4; 1879 rdev->config.evergreen.max_simds = 2; 1880 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1881 rdev->config.evergreen.max_gprs = 256; 1882 rdev->config.evergreen.max_threads = 248; 1883 rdev->config.evergreen.max_gs_threads = 32; 1884 rdev->config.evergreen.max_stack_entries = 512; 1885 rdev->config.evergreen.sx_num_of_sets = 4; 1886 rdev->config.evergreen.sx_max_export_size = 256; 1887 rdev->config.evergreen.sx_max_export_pos_size = 64; 1888 rdev->config.evergreen.sx_max_export_smx_size = 192; 1889 rdev->config.evergreen.max_hw_contexts = 8; 1890 rdev->config.evergreen.sq_num_cf_insts = 2; 1891 1892 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1893 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1894 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1895 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN; 1896 break; 1897 case CHIP_BARTS: 1898 rdev->config.evergreen.num_ses = 2; 1899 rdev->config.evergreen.max_pipes = 4; 1900 rdev->config.evergreen.max_tile_pipes = 8; 1901 rdev->config.evergreen.max_simds = 7; 1902 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 1903 rdev->config.evergreen.max_gprs = 256; 1904 rdev->config.evergreen.max_threads = 248; 1905 rdev->config.evergreen.max_gs_threads = 32; 1906 rdev->config.evergreen.max_stack_entries = 512; 1907 rdev->config.evergreen.sx_num_of_sets = 4; 1908 rdev->config.evergreen.sx_max_export_size = 256; 1909 rdev->config.evergreen.sx_max_export_pos_size = 64; 1910 rdev->config.evergreen.sx_max_export_smx_size = 192; 1911 rdev->config.evergreen.max_hw_contexts = 8; 1912 rdev->config.evergreen.sq_num_cf_insts = 2; 1913 1914 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1915 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1916 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1917 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN; 1918 break; 1919 case CHIP_TURKS: 1920 rdev->config.evergreen.num_ses = 1; 1921 rdev->config.evergreen.max_pipes = 4; 1922 rdev->config.evergreen.max_tile_pipes = 4; 1923 rdev->config.evergreen.max_simds = 6; 1924 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 1925 rdev->config.evergreen.max_gprs = 256; 1926 rdev->config.evergreen.max_threads = 248; 1927 rdev->config.evergreen.max_gs_threads = 32; 1928 rdev->config.evergreen.max_stack_entries = 256; 1929 rdev->config.evergreen.sx_num_of_sets = 4; 1930 rdev->config.evergreen.sx_max_export_size = 256; 1931 rdev->config.evergreen.sx_max_export_pos_size = 64; 1932 rdev->config.evergreen.sx_max_export_smx_size = 192; 1933 rdev->config.evergreen.max_hw_contexts = 8; 1934 rdev->config.evergreen.sq_num_cf_insts = 2; 1935 1936 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1937 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1938 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1939 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN; 1940 break; 1941 case CHIP_CAICOS: 1942 rdev->config.evergreen.num_ses = 1; 1943 rdev->config.evergreen.max_pipes = 2; 1944 rdev->config.evergreen.max_tile_pipes = 2; 1945 rdev->config.evergreen.max_simds = 2; 1946 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1947 rdev->config.evergreen.max_gprs = 256; 1948 rdev->config.evergreen.max_threads = 192; 1949 rdev->config.evergreen.max_gs_threads = 16; 1950 rdev->config.evergreen.max_stack_entries = 256; 1951 rdev->config.evergreen.sx_num_of_sets = 4; 1952 rdev->config.evergreen.sx_max_export_size = 128; 1953 rdev->config.evergreen.sx_max_export_pos_size = 32; 1954 rdev->config.evergreen.sx_max_export_smx_size = 96; 1955 rdev->config.evergreen.max_hw_contexts = 4; 1956 rdev->config.evergreen.sq_num_cf_insts = 1; 1957 1958 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1959 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1960 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1961 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN; 1962 break; 1963 } 1964 1965 /* Initialize HDP */ 1966 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 1967 WREG32((0x2c14 + j), 0x00000000); 1968 WREG32((0x2c18 + j), 0x00000000); 1969 WREG32((0x2c1c + j), 0x00000000); 1970 WREG32((0x2c20 + j), 0x00000000); 1971 WREG32((0x2c24 + j), 0x00000000); 1972 } 1973 1974 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 1975 1976 evergreen_fix_pci_max_read_req_size(rdev); 1977 1978 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 1979 if ((rdev->family == CHIP_PALM) || 1980 (rdev->family == CHIP_SUMO) || 1981 (rdev->family == CHIP_SUMO2)) 1982 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 1983 else 1984 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 1985 1986 /* setup tiling info dword. gb_addr_config is not adequate since it does 1987 * not have bank info, so create a custom tiling dword. 1988 * bits 3:0 num_pipes 1989 * bits 7:4 num_banks 1990 * bits 11:8 group_size 1991 * bits 15:12 row_size 1992 */ 1993 rdev->config.evergreen.tile_config = 0; 1994 switch (rdev->config.evergreen.max_tile_pipes) { 1995 case 1: 1996 default: 1997 rdev->config.evergreen.tile_config |= (0 << 0); 1998 break; 1999 case 2: 2000 rdev->config.evergreen.tile_config |= (1 << 0); 2001 break; 2002 case 4: 2003 rdev->config.evergreen.tile_config |= (2 << 0); 2004 break; 2005 case 8: 2006 rdev->config.evergreen.tile_config |= (3 << 0); 2007 break; 2008 } 2009 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 2010 if (rdev->flags & RADEON_IS_IGP) 2011 rdev->config.evergreen.tile_config |= 1 << 4; 2012 else { 2013 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 2014 case 0: /* four banks */ 2015 rdev->config.evergreen.tile_config |= 0 << 4; 2016 break; 2017 case 1: /* eight banks */ 2018 rdev->config.evergreen.tile_config |= 1 << 4; 2019 break; 2020 case 2: /* sixteen banks */ 2021 default: 2022 rdev->config.evergreen.tile_config |= 2 << 4; 2023 break; 2024 } 2025 } 2026 rdev->config.evergreen.tile_config |= 0 << 8; 2027 rdev->config.evergreen.tile_config |= 2028 ((gb_addr_config & 0x30000000) >> 28) << 12; 2029 2030 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1; 2031 2032 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { 2033 u32 efuse_straps_4; 2034 u32 efuse_straps_3; 2035 2036 WREG32(RCU_IND_INDEX, 0x204); 2037 efuse_straps_4 = RREG32(RCU_IND_DATA); 2038 WREG32(RCU_IND_INDEX, 0x203); 2039 efuse_straps_3 = RREG32(RCU_IND_DATA); 2040 tmp = (((efuse_straps_4 & 0xf) << 4) | 2041 ((efuse_straps_3 & 0xf0000000) >> 28)); 2042 } else { 2043 tmp = 0; 2044 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) { 2045 u32 rb_disable_bitmap; 2046 2047 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 2048 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 2049 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 2050 tmp <<= 4; 2051 tmp |= rb_disable_bitmap; 2052 } 2053 } 2054 /* enabled rb are just the one not disabled :) */ 2055 disabled_rb_mask = tmp; 2056 2057 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 2058 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 2059 2060 WREG32(GB_ADDR_CONFIG, gb_addr_config); 2061 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 2062 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 2063 WREG32(DMA_TILING_CONFIG, gb_addr_config); 2064 2065 if ((rdev->config.evergreen.max_backends == 1) && 2066 (rdev->flags & RADEON_IS_IGP)) { 2067 if ((disabled_rb_mask & 3) == 1) { 2068 /* RB0 disabled, RB1 enabled */ 2069 tmp = 0x11111111; 2070 } else { 2071 /* RB1 disabled, RB0 enabled */ 2072 tmp = 0x00000000; 2073 } 2074 } else { 2075 tmp = gb_addr_config & NUM_PIPES_MASK; 2076 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends, 2077 EVERGREEN_MAX_BACKENDS, disabled_rb_mask); 2078 } 2079 WREG32(GB_BACKEND_MAP, tmp); 2080 2081 WREG32(CGTS_SYS_TCC_DISABLE, 0); 2082 WREG32(CGTS_TCC_DISABLE, 0); 2083 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 2084 WREG32(CGTS_USER_TCC_DISABLE, 0); 2085 2086 /* set HW defaults for 3D engine */ 2087 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 2088 ROQ_IB2_START(0x2b))); 2089 2090 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 2091 2092 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 2093 SYNC_GRADIENT | 2094 SYNC_WALKER | 2095 SYNC_ALIGNER)); 2096 2097 sx_debug_1 = RREG32(SX_DEBUG_1); 2098 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 2099 WREG32(SX_DEBUG_1, sx_debug_1); 2100 2101 2102 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 2103 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 2104 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 2105 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 2106 2107 if (rdev->family <= CHIP_SUMO2) 2108 WREG32(SMX_SAR_CTL0, 0x00010000); 2109 2110 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 2111 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 2112 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 2113 2114 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 2115 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 2116 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 2117 2118 WREG32(VGT_NUM_INSTANCES, 1); 2119 WREG32(SPI_CONFIG_CNTL, 0); 2120 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 2121 WREG32(CP_PERFMON_CNTL, 0); 2122 2123 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 2124 FETCH_FIFO_HIWATER(0x4) | 2125 DONE_FIFO_HIWATER(0xe0) | 2126 ALU_UPDATE_FIFO_HIWATER(0x8))); 2127 2128 sq_config = RREG32(SQ_CONFIG); 2129 sq_config &= ~(PS_PRIO(3) | 2130 VS_PRIO(3) | 2131 GS_PRIO(3) | 2132 ES_PRIO(3)); 2133 sq_config |= (VC_ENABLE | 2134 EXPORT_SRC_C | 2135 PS_PRIO(0) | 2136 VS_PRIO(1) | 2137 GS_PRIO(2) | 2138 ES_PRIO(3)); 2139 2140 switch (rdev->family) { 2141 case CHIP_CEDAR: 2142 case CHIP_PALM: 2143 case CHIP_SUMO: 2144 case CHIP_SUMO2: 2145 case CHIP_CAICOS: 2146 /* no vertex cache */ 2147 sq_config &= ~VC_ENABLE; 2148 break; 2149 default: 2150 break; 2151 } 2152 2153 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 2154 2155 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 2156 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 2157 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 2158 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 2159 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 2160 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 2161 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 2162 2163 switch (rdev->family) { 2164 case CHIP_CEDAR: 2165 case CHIP_PALM: 2166 case CHIP_SUMO: 2167 case CHIP_SUMO2: 2168 ps_thread_count = 96; 2169 break; 2170 default: 2171 ps_thread_count = 128; 2172 break; 2173 } 2174 2175 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 2176 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2177 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2178 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2179 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2180 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2181 2182 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2183 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2184 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2185 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2186 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2187 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2188 2189 WREG32(SQ_CONFIG, sq_config); 2190 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 2191 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 2192 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 2193 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 2194 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 2195 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 2196 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 2197 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 2198 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 2199 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 2200 2201 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 2202 FORCE_EOV_MAX_REZ_CNT(255))); 2203 2204 switch (rdev->family) { 2205 case CHIP_CEDAR: 2206 case CHIP_PALM: 2207 case CHIP_SUMO: 2208 case CHIP_SUMO2: 2209 case CHIP_CAICOS: 2210 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 2211 break; 2212 default: 2213 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 2214 break; 2215 } 2216 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 2217 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 2218 2219 WREG32(VGT_GS_VERTEX_REUSE, 16); 2220 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 2221 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 2222 2223 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 2224 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 2225 2226 WREG32(CB_PERF_CTR0_SEL_0, 0); 2227 WREG32(CB_PERF_CTR0_SEL_1, 0); 2228 WREG32(CB_PERF_CTR1_SEL_0, 0); 2229 WREG32(CB_PERF_CTR1_SEL_1, 0); 2230 WREG32(CB_PERF_CTR2_SEL_0, 0); 2231 WREG32(CB_PERF_CTR2_SEL_1, 0); 2232 WREG32(CB_PERF_CTR3_SEL_0, 0); 2233 WREG32(CB_PERF_CTR3_SEL_1, 0); 2234 2235 /* clear render buffer base addresses */ 2236 WREG32(CB_COLOR0_BASE, 0); 2237 WREG32(CB_COLOR1_BASE, 0); 2238 WREG32(CB_COLOR2_BASE, 0); 2239 WREG32(CB_COLOR3_BASE, 0); 2240 WREG32(CB_COLOR4_BASE, 0); 2241 WREG32(CB_COLOR5_BASE, 0); 2242 WREG32(CB_COLOR6_BASE, 0); 2243 WREG32(CB_COLOR7_BASE, 0); 2244 WREG32(CB_COLOR8_BASE, 0); 2245 WREG32(CB_COLOR9_BASE, 0); 2246 WREG32(CB_COLOR10_BASE, 0); 2247 WREG32(CB_COLOR11_BASE, 0); 2248 2249 /* set the shader const cache sizes to 0 */ 2250 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 2251 WREG32(i, 0); 2252 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 2253 WREG32(i, 0); 2254 2255 tmp = RREG32(HDP_MISC_CNTL); 2256 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 2257 WREG32(HDP_MISC_CNTL, tmp); 2258 2259 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 2260 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 2261 2262 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 2263 2264 DRM_UDELAY(50); 2265 2266 } 2267 2268 int evergreen_mc_init(struct radeon_device *rdev) 2269 { 2270 u32 tmp; 2271 int chansize, numchan; 2272 2273 /* Get VRAM informations */ 2274 rdev->mc.vram_is_ddr = true; 2275 if ((rdev->family == CHIP_PALM) || 2276 (rdev->family == CHIP_SUMO) || 2277 (rdev->family == CHIP_SUMO2)) 2278 tmp = RREG32(FUS_MC_ARB_RAMCFG); 2279 else 2280 tmp = RREG32(MC_ARB_RAMCFG); 2281 if (tmp & CHANSIZE_OVERRIDE) { 2282 chansize = 16; 2283 } else if (tmp & CHANSIZE_MASK) { 2284 chansize = 64; 2285 } else { 2286 chansize = 32; 2287 } 2288 tmp = RREG32(MC_SHARED_CHMAP); 2289 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 2290 case 0: 2291 default: 2292 numchan = 1; 2293 break; 2294 case 1: 2295 numchan = 2; 2296 break; 2297 case 2: 2298 numchan = 4; 2299 break; 2300 case 3: 2301 numchan = 8; 2302 break; 2303 } 2304 rdev->mc.vram_width = numchan * chansize; 2305 /* Could aper size report 0 ? */ 2306 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0); 2307 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0); 2308 /* Setup GPU memory space */ 2309 if ((rdev->family == CHIP_PALM) || 2310 (rdev->family == CHIP_SUMO) || 2311 (rdev->family == CHIP_SUMO2)) { 2312 /* size in bytes on fusion */ 2313 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 2314 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 2315 } else { 2316 /* size in MB on evergreen/cayman/tn */ 2317 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2318 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2319 } 2320 rdev->mc.visible_vram_size = rdev->mc.aper_size; 2321 r700_vram_gtt_location(rdev, &rdev->mc); 2322 radeon_update_bandwidth_info(rdev); 2323 2324 return 0; 2325 } 2326 2327 void evergreen_print_gpu_status_regs(struct radeon_device *rdev) 2328 { 2329 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", 2330 RREG32(GRBM_STATUS)); 2331 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", 2332 RREG32(GRBM_STATUS_SE0)); 2333 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", 2334 RREG32(GRBM_STATUS_SE1)); 2335 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", 2336 RREG32(SRBM_STATUS)); 2337 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", 2338 RREG32(SRBM_STATUS2)); 2339 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", 2340 RREG32(CP_STALLED_STAT1)); 2341 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", 2342 RREG32(CP_STALLED_STAT2)); 2343 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", 2344 RREG32(CP_BUSY_STAT)); 2345 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", 2346 RREG32(CP_STAT)); 2347 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", 2348 RREG32(DMA_STATUS_REG)); 2349 if (rdev->family >= CHIP_CAYMAN) { 2350 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", 2351 RREG32(DMA_STATUS_REG + 0x800)); 2352 } 2353 } 2354 2355 bool evergreen_is_display_hung(struct radeon_device *rdev) 2356 { 2357 u32 crtc_hung = 0; 2358 u32 crtc_status[6]; 2359 u32 i, j, tmp; 2360 2361 for (i = 0; i < rdev->num_crtc; i++) { 2362 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { 2363 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 2364 crtc_hung |= (1 << i); 2365 } 2366 } 2367 2368 for (j = 0; j < 10; j++) { 2369 for (i = 0; i < rdev->num_crtc; i++) { 2370 if (crtc_hung & (1 << i)) { 2371 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 2372 if (tmp != crtc_status[i]) 2373 crtc_hung &= ~(1 << i); 2374 } 2375 } 2376 if (crtc_hung == 0) 2377 return false; 2378 DRM_UDELAY(100); 2379 } 2380 2381 return true; 2382 } 2383 2384 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) 2385 { 2386 u32 reset_mask = 0; 2387 u32 tmp; 2388 2389 /* GRBM_STATUS */ 2390 tmp = RREG32(GRBM_STATUS); 2391 if (tmp & (PA_BUSY | SC_BUSY | 2392 SH_BUSY | SX_BUSY | 2393 TA_BUSY | VGT_BUSY | 2394 DB_BUSY | CB_BUSY | 2395 SPI_BUSY | VGT_BUSY_NO_DMA)) 2396 reset_mask |= RADEON_RESET_GFX; 2397 2398 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 2399 CP_BUSY | CP_COHERENCY_BUSY)) 2400 reset_mask |= RADEON_RESET_CP; 2401 2402 if (tmp & GRBM_EE_BUSY) 2403 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 2404 2405 /* DMA_STATUS_REG */ 2406 tmp = RREG32(DMA_STATUS_REG); 2407 if (!(tmp & DMA_IDLE)) 2408 reset_mask |= RADEON_RESET_DMA; 2409 2410 /* SRBM_STATUS2 */ 2411 tmp = RREG32(SRBM_STATUS2); 2412 if (tmp & DMA_BUSY) 2413 reset_mask |= RADEON_RESET_DMA; 2414 2415 /* SRBM_STATUS */ 2416 tmp = RREG32(SRBM_STATUS); 2417 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 2418 reset_mask |= RADEON_RESET_RLC; 2419 2420 if (tmp & IH_BUSY) 2421 reset_mask |= RADEON_RESET_IH; 2422 2423 if (tmp & SEM_BUSY) 2424 reset_mask |= RADEON_RESET_SEM; 2425 2426 if (tmp & GRBM_RQ_PENDING) 2427 reset_mask |= RADEON_RESET_GRBM; 2428 2429 if (tmp & VMC_BUSY) 2430 reset_mask |= RADEON_RESET_VMC; 2431 2432 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 2433 MCC_BUSY | MCD_BUSY)) 2434 reset_mask |= RADEON_RESET_MC; 2435 2436 if (evergreen_is_display_hung(rdev)) 2437 reset_mask |= RADEON_RESET_DISPLAY; 2438 2439 /* VM_L2_STATUS */ 2440 tmp = RREG32(VM_L2_STATUS); 2441 if (tmp & L2_BUSY) 2442 reset_mask |= RADEON_RESET_VMC; 2443 2444 /* Skip MC reset as it's mostly likely not hung, just busy */ 2445 if (reset_mask & RADEON_RESET_MC) { 2446 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 2447 reset_mask &= ~RADEON_RESET_MC; 2448 } 2449 2450 return reset_mask; 2451 } 2452 2453 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 2454 { 2455 struct evergreen_mc_save save; 2456 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 2457 u32 tmp; 2458 2459 if (reset_mask == 0) 2460 return; 2461 2462 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 2463 2464 evergreen_print_gpu_status_regs(rdev); 2465 2466 /* Disable CP parsing/prefetching */ 2467 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 2468 2469 if (reset_mask & RADEON_RESET_DMA) { 2470 /* Disable DMA */ 2471 tmp = RREG32(DMA_RB_CNTL); 2472 tmp &= ~DMA_RB_ENABLE; 2473 WREG32(DMA_RB_CNTL, tmp); 2474 } 2475 2476 DRM_UDELAY(50); 2477 2478 evergreen_mc_stop(rdev, &save); 2479 if (evergreen_mc_wait_for_idle(rdev)) { 2480 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2481 } 2482 2483 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 2484 grbm_soft_reset |= SOFT_RESET_DB | 2485 SOFT_RESET_CB | 2486 SOFT_RESET_PA | 2487 SOFT_RESET_SC | 2488 SOFT_RESET_SPI | 2489 SOFT_RESET_SX | 2490 SOFT_RESET_SH | 2491 SOFT_RESET_TC | 2492 SOFT_RESET_TA | 2493 SOFT_RESET_VC | 2494 SOFT_RESET_VGT; 2495 } 2496 2497 if (reset_mask & RADEON_RESET_CP) { 2498 grbm_soft_reset |= SOFT_RESET_CP | 2499 SOFT_RESET_VGT; 2500 2501 srbm_soft_reset |= SOFT_RESET_GRBM; 2502 } 2503 2504 if (reset_mask & RADEON_RESET_DMA) 2505 srbm_soft_reset |= SOFT_RESET_DMA; 2506 2507 if (reset_mask & RADEON_RESET_DISPLAY) 2508 srbm_soft_reset |= SOFT_RESET_DC; 2509 2510 if (reset_mask & RADEON_RESET_RLC) 2511 srbm_soft_reset |= SOFT_RESET_RLC; 2512 2513 if (reset_mask & RADEON_RESET_SEM) 2514 srbm_soft_reset |= SOFT_RESET_SEM; 2515 2516 if (reset_mask & RADEON_RESET_IH) 2517 srbm_soft_reset |= SOFT_RESET_IH; 2518 2519 if (reset_mask & RADEON_RESET_GRBM) 2520 srbm_soft_reset |= SOFT_RESET_GRBM; 2521 2522 if (reset_mask & RADEON_RESET_VMC) 2523 srbm_soft_reset |= SOFT_RESET_VMC; 2524 2525 if (!(rdev->flags & RADEON_IS_IGP)) { 2526 if (reset_mask & RADEON_RESET_MC) 2527 srbm_soft_reset |= SOFT_RESET_MC; 2528 } 2529 2530 if (grbm_soft_reset) { 2531 tmp = RREG32(GRBM_SOFT_RESET); 2532 tmp |= grbm_soft_reset; 2533 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 2534 WREG32(GRBM_SOFT_RESET, tmp); 2535 tmp = RREG32(GRBM_SOFT_RESET); 2536 2537 DRM_UDELAY(50); 2538 2539 tmp &= ~grbm_soft_reset; 2540 WREG32(GRBM_SOFT_RESET, tmp); 2541 tmp = RREG32(GRBM_SOFT_RESET); 2542 } 2543 2544 if (srbm_soft_reset) { 2545 tmp = RREG32(SRBM_SOFT_RESET); 2546 tmp |= srbm_soft_reset; 2547 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 2548 WREG32(SRBM_SOFT_RESET, tmp); 2549 tmp = RREG32(SRBM_SOFT_RESET); 2550 2551 DRM_UDELAY(50); 2552 2553 tmp &= ~srbm_soft_reset; 2554 WREG32(SRBM_SOFT_RESET, tmp); 2555 tmp = RREG32(SRBM_SOFT_RESET); 2556 } 2557 2558 /* Wait a little for things to settle down */ 2559 DRM_UDELAY(50); 2560 2561 evergreen_mc_resume(rdev, &save); 2562 DRM_UDELAY(50); 2563 2564 evergreen_print_gpu_status_regs(rdev); 2565 } 2566 2567 int evergreen_asic_reset(struct radeon_device *rdev) 2568 { 2569 u32 reset_mask; 2570 2571 reset_mask = evergreen_gpu_check_soft_reset(rdev); 2572 2573 if (reset_mask) 2574 r600_set_bios_scratch_engine_hung(rdev, true); 2575 2576 evergreen_gpu_soft_reset(rdev, reset_mask); 2577 2578 reset_mask = evergreen_gpu_check_soft_reset(rdev); 2579 2580 if (!reset_mask) 2581 r600_set_bios_scratch_engine_hung(rdev, false); 2582 2583 return 0; 2584 } 2585 2586 /** 2587 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up 2588 * 2589 * @rdev: radeon_device pointer 2590 * @ring: radeon_ring structure holding ring information 2591 * 2592 * Check if the GFX engine is locked up. 2593 * Returns true if the engine appears to be locked up, false if not. 2594 */ 2595 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 2596 { 2597 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 2598 2599 if (!(reset_mask & (RADEON_RESET_GFX | 2600 RADEON_RESET_COMPUTE | 2601 RADEON_RESET_CP))) { 2602 radeon_ring_lockup_update(ring); 2603 return false; 2604 } 2605 /* force CP activities */ 2606 radeon_ring_force_activity(rdev, ring); 2607 return radeon_ring_test_lockup(rdev, ring); 2608 } 2609 2610 /** 2611 * evergreen_dma_is_lockup - Check if the DMA engine is locked up 2612 * 2613 * @rdev: radeon_device pointer 2614 * @ring: radeon_ring structure holding ring information 2615 * 2616 * Check if the async DMA engine is locked up. 2617 * Returns true if the engine appears to be locked up, false if not. 2618 */ 2619 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 2620 { 2621 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 2622 2623 if (!(reset_mask & RADEON_RESET_DMA)) { 2624 radeon_ring_lockup_update(ring); 2625 return false; 2626 } 2627 /* force ring activities */ 2628 radeon_ring_force_activity(rdev, ring); 2629 return radeon_ring_test_lockup(rdev, ring); 2630 } 2631 2632 /* Interrupts */ 2633 2634 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 2635 { 2636 if (crtc >= rdev->num_crtc) 2637 return 0; 2638 else 2639 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); 2640 } 2641 2642 void evergreen_disable_interrupt_state(struct radeon_device *rdev) 2643 { 2644 u32 tmp; 2645 2646 if (rdev->family >= CHIP_CAYMAN) { 2647 cayman_cp_int_cntl_setup(rdev, 0, 2648 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 2649 cayman_cp_int_cntl_setup(rdev, 1, 0); 2650 cayman_cp_int_cntl_setup(rdev, 2, 0); 2651 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 2652 WREG32(CAYMAN_DMA1_CNTL, tmp); 2653 } else 2654 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 2655 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 2656 WREG32(DMA_CNTL, tmp); 2657 WREG32(GRBM_INT_CNTL, 0); 2658 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 2659 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 2660 if (rdev->num_crtc >= 4) { 2661 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 2662 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 2663 } 2664 if (rdev->num_crtc >= 6) { 2665 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 2666 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 2667 } 2668 2669 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 2670 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 2671 if (rdev->num_crtc >= 4) { 2672 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 2673 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 2674 } 2675 if (rdev->num_crtc >= 6) { 2676 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 2677 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 2678 } 2679 2680 /* only one DAC on DCE6 */ 2681 if (!ASIC_IS_DCE6(rdev)) 2682 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 2683 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 2684 2685 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2686 WREG32(DC_HPD1_INT_CONTROL, tmp); 2687 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2688 WREG32(DC_HPD2_INT_CONTROL, tmp); 2689 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2690 WREG32(DC_HPD3_INT_CONTROL, tmp); 2691 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2692 WREG32(DC_HPD4_INT_CONTROL, tmp); 2693 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2694 WREG32(DC_HPD5_INT_CONTROL, tmp); 2695 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2696 WREG32(DC_HPD6_INT_CONTROL, tmp); 2697 2698 } 2699 2700 int evergreen_irq_set(struct radeon_device *rdev) 2701 { 2702 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 2703 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 2704 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 2705 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 2706 u32 grbm_int_cntl = 0; 2707 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; 2708 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; 2709 u32 dma_cntl, dma_cntl1 = 0; 2710 2711 if (!rdev->irq.installed) { 2712 dev_warn(rdev->dev, "Can't enable IRQ/MSI because no handler is installed\n"); 2713 return -EINVAL; 2714 } 2715 /* don't enable anything if the ih is disabled */ 2716 if (!rdev->ih.enabled) { 2717 r600_disable_interrupts(rdev); 2718 /* force the active interrupt state to all disabled */ 2719 evergreen_disable_interrupt_state(rdev); 2720 return 0; 2721 } 2722 2723 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 2724 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 2725 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 2726 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 2727 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 2728 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 2729 2730 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 2731 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 2732 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 2733 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 2734 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 2735 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 2736 2737 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 2738 2739 if (rdev->family >= CHIP_CAYMAN) { 2740 /* enable CP interrupts on all rings */ 2741 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 2742 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 2743 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 2744 } 2745 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 2746 DRM_DEBUG("evergreen_irq_set: sw int cp1\n"); 2747 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 2748 } 2749 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 2750 DRM_DEBUG("evergreen_irq_set: sw int cp2\n"); 2751 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 2752 } 2753 } else { 2754 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 2755 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 2756 cp_int_cntl |= RB_INT_ENABLE; 2757 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 2758 } 2759 } 2760 2761 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 2762 DRM_DEBUG("r600_irq_set: sw int dma\n"); 2763 dma_cntl |= TRAP_ENABLE; 2764 } 2765 2766 if (rdev->family >= CHIP_CAYMAN) { 2767 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 2768 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 2769 DRM_DEBUG("r600_irq_set: sw int dma1\n"); 2770 dma_cntl1 |= TRAP_ENABLE; 2771 } 2772 } 2773 2774 if (rdev->irq.crtc_vblank_int[0] || 2775 atomic_read(&rdev->irq.pflip[0])) { 2776 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 2777 crtc1 |= VBLANK_INT_MASK; 2778 } 2779 if (rdev->irq.crtc_vblank_int[1] || 2780 atomic_read(&rdev->irq.pflip[1])) { 2781 DRM_DEBUG("evergreen_irq_set: vblank 1\n"); 2782 crtc2 |= VBLANK_INT_MASK; 2783 } 2784 if (rdev->irq.crtc_vblank_int[2] || 2785 atomic_read(&rdev->irq.pflip[2])) { 2786 DRM_DEBUG("evergreen_irq_set: vblank 2\n"); 2787 crtc3 |= VBLANK_INT_MASK; 2788 } 2789 if (rdev->irq.crtc_vblank_int[3] || 2790 atomic_read(&rdev->irq.pflip[3])) { 2791 DRM_DEBUG("evergreen_irq_set: vblank 3\n"); 2792 crtc4 |= VBLANK_INT_MASK; 2793 } 2794 if (rdev->irq.crtc_vblank_int[4] || 2795 atomic_read(&rdev->irq.pflip[4])) { 2796 DRM_DEBUG("evergreen_irq_set: vblank 4\n"); 2797 crtc5 |= VBLANK_INT_MASK; 2798 } 2799 if (rdev->irq.crtc_vblank_int[5] || 2800 atomic_read(&rdev->irq.pflip[5])) { 2801 DRM_DEBUG("evergreen_irq_set: vblank 5\n"); 2802 crtc6 |= VBLANK_INT_MASK; 2803 } 2804 if (rdev->irq.hpd[0]) { 2805 DRM_DEBUG("evergreen_irq_set: hpd 1\n"); 2806 hpd1 |= DC_HPDx_INT_EN; 2807 } 2808 if (rdev->irq.hpd[1]) { 2809 DRM_DEBUG("evergreen_irq_set: hpd 2\n"); 2810 hpd2 |= DC_HPDx_INT_EN; 2811 } 2812 if (rdev->irq.hpd[2]) { 2813 DRM_DEBUG("evergreen_irq_set: hpd 3\n"); 2814 hpd3 |= DC_HPDx_INT_EN; 2815 } 2816 if (rdev->irq.hpd[3]) { 2817 DRM_DEBUG("evergreen_irq_set: hpd 4\n"); 2818 hpd4 |= DC_HPDx_INT_EN; 2819 } 2820 if (rdev->irq.hpd[4]) { 2821 DRM_DEBUG("evergreen_irq_set: hpd 5\n"); 2822 hpd5 |= DC_HPDx_INT_EN; 2823 } 2824 if (rdev->irq.hpd[5]) { 2825 DRM_DEBUG("evergreen_irq_set: hpd 6\n"); 2826 hpd6 |= DC_HPDx_INT_EN; 2827 } 2828 if (rdev->irq.afmt[0]) { 2829 DRM_DEBUG("evergreen_irq_set: hdmi 0\n"); 2830 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK; 2831 } 2832 if (rdev->irq.afmt[1]) { 2833 DRM_DEBUG("evergreen_irq_set: hdmi 1\n"); 2834 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK; 2835 } 2836 if (rdev->irq.afmt[2]) { 2837 DRM_DEBUG("evergreen_irq_set: hdmi 2\n"); 2838 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK; 2839 } 2840 if (rdev->irq.afmt[3]) { 2841 DRM_DEBUG("evergreen_irq_set: hdmi 3\n"); 2842 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK; 2843 } 2844 if (rdev->irq.afmt[4]) { 2845 DRM_DEBUG("evergreen_irq_set: hdmi 4\n"); 2846 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK; 2847 } 2848 if (rdev->irq.afmt[5]) { 2849 DRM_DEBUG("evergreen_irq_set: hdmi 5\n"); 2850 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK; 2851 } 2852 2853 if (rdev->family >= CHIP_CAYMAN) { 2854 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl); 2855 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1); 2856 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2); 2857 } else 2858 WREG32(CP_INT_CNTL, cp_int_cntl); 2859 2860 WREG32(DMA_CNTL, dma_cntl); 2861 2862 if (rdev->family >= CHIP_CAYMAN) 2863 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1); 2864 2865 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 2866 2867 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 2868 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 2869 if (rdev->num_crtc >= 4) { 2870 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 2871 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 2872 } 2873 if (rdev->num_crtc >= 6) { 2874 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 2875 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 2876 } 2877 2878 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); 2879 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); 2880 if (rdev->num_crtc >= 4) { 2881 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); 2882 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); 2883 } 2884 if (rdev->num_crtc >= 6) { 2885 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); 2886 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); 2887 } 2888 2889 WREG32(DC_HPD1_INT_CONTROL, hpd1); 2890 WREG32(DC_HPD2_INT_CONTROL, hpd2); 2891 WREG32(DC_HPD3_INT_CONTROL, hpd3); 2892 WREG32(DC_HPD4_INT_CONTROL, hpd4); 2893 WREG32(DC_HPD5_INT_CONTROL, hpd5); 2894 WREG32(DC_HPD6_INT_CONTROL, hpd6); 2895 2896 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1); 2897 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2); 2898 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3); 2899 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4); 2900 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5); 2901 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6); 2902 2903 return 0; 2904 } 2905 2906 static void evergreen_irq_ack(struct radeon_device *rdev) 2907 { 2908 u32 tmp; 2909 2910 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 2911 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 2912 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 2913 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 2914 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 2915 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 2916 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 2917 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 2918 if (rdev->num_crtc >= 4) { 2919 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 2920 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 2921 } 2922 if (rdev->num_crtc >= 6) { 2923 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 2924 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 2925 } 2926 2927 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 2928 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 2929 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 2930 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 2931 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 2932 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 2933 2934 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 2935 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2936 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 2937 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2938 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 2939 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 2940 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 2941 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 2942 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 2943 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 2944 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 2945 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 2946 2947 if (rdev->num_crtc >= 4) { 2948 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 2949 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2950 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 2951 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2952 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 2953 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 2954 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 2955 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 2956 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 2957 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 2958 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 2959 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 2960 } 2961 2962 if (rdev->num_crtc >= 6) { 2963 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 2964 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2965 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 2966 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2967 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 2968 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 2969 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 2970 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 2971 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 2972 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 2973 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 2974 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 2975 } 2976 2977 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 2978 tmp = RREG32(DC_HPD1_INT_CONTROL); 2979 tmp |= DC_HPDx_INT_ACK; 2980 WREG32(DC_HPD1_INT_CONTROL, tmp); 2981 } 2982 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 2983 tmp = RREG32(DC_HPD2_INT_CONTROL); 2984 tmp |= DC_HPDx_INT_ACK; 2985 WREG32(DC_HPD2_INT_CONTROL, tmp); 2986 } 2987 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 2988 tmp = RREG32(DC_HPD3_INT_CONTROL); 2989 tmp |= DC_HPDx_INT_ACK; 2990 WREG32(DC_HPD3_INT_CONTROL, tmp); 2991 } 2992 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 2993 tmp = RREG32(DC_HPD4_INT_CONTROL); 2994 tmp |= DC_HPDx_INT_ACK; 2995 WREG32(DC_HPD4_INT_CONTROL, tmp); 2996 } 2997 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 2998 tmp = RREG32(DC_HPD5_INT_CONTROL); 2999 tmp |= DC_HPDx_INT_ACK; 3000 WREG32(DC_HPD5_INT_CONTROL, tmp); 3001 } 3002 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 3003 tmp = RREG32(DC_HPD5_INT_CONTROL); 3004 tmp |= DC_HPDx_INT_ACK; 3005 WREG32(DC_HPD6_INT_CONTROL, tmp); 3006 } 3007 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 3008 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); 3009 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 3010 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp); 3011 } 3012 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 3013 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); 3014 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 3015 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp); 3016 } 3017 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 3018 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET); 3019 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 3020 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp); 3021 } 3022 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 3023 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); 3024 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 3025 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp); 3026 } 3027 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 3028 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET); 3029 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 3030 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp); 3031 } 3032 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 3033 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 3034 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 3035 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp); 3036 } 3037 } 3038 3039 static void evergreen_irq_disable(struct radeon_device *rdev) 3040 { 3041 r600_disable_interrupts(rdev); 3042 /* Wait and acknowledge irq */ 3043 DRM_MDELAY(1); 3044 evergreen_irq_ack(rdev); 3045 evergreen_disable_interrupt_state(rdev); 3046 } 3047 3048 void evergreen_irq_suspend(struct radeon_device *rdev) 3049 { 3050 evergreen_irq_disable(rdev); 3051 r600_rlc_stop(rdev); 3052 } 3053 3054 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 3055 { 3056 u32 wptr, tmp; 3057 3058 if (rdev->wb.enabled) 3059 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 3060 else 3061 wptr = RREG32(IH_RB_WPTR); 3062 3063 if (wptr & RB_OVERFLOW) { 3064 /* When a ring buffer overflow happen start parsing interrupt 3065 * from the last not overwritten vector (wptr + 16). Hopefully 3066 * this should allow us to catchup. 3067 */ 3068 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 3069 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 3070 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 3071 tmp = RREG32(IH_RB_CNTL); 3072 tmp |= IH_WPTR_OVERFLOW_CLEAR; 3073 WREG32(IH_RB_CNTL, tmp); 3074 } 3075 return (wptr & rdev->ih.ptr_mask); 3076 } 3077 3078 irqreturn_t evergreen_irq_process(struct radeon_device *rdev) 3079 { 3080 u32 wptr; 3081 u32 rptr; 3082 u32 src_id, src_data; 3083 u32 ring_index; 3084 bool queue_hotplug = false; 3085 bool queue_hdmi = false; 3086 3087 if (!rdev->ih.enabled || rdev->shutdown) 3088 return IRQ_NONE; 3089 3090 wptr = evergreen_get_ih_wptr(rdev); 3091 3092 restart_ih: 3093 /* is somebody else already processing irqs? */ 3094 if (atomic_xchg(&rdev->ih.lock, 1)) 3095 return IRQ_NONE; 3096 3097 rptr = rdev->ih.rptr; 3098 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 3099 3100 /* Order reading of wptr vs. reading of IH ring data */ 3101 cpu_lfence(); 3102 3103 /* display interrupts */ 3104 evergreen_irq_ack(rdev); 3105 3106 while (rptr != wptr) { 3107 /* wptr/rptr are in bytes! */ 3108 ring_index = rptr / 4; 3109 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 3110 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 3111 3112 switch (src_id) { 3113 case 1: /* D1 vblank/vline */ 3114 switch (src_data) { 3115 case 0: /* D1 vblank */ 3116 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 3117 if (rdev->irq.crtc_vblank_int[0]) { 3118 drm_handle_vblank(rdev->ddev, 0); 3119 rdev->pm.vblank_sync = true; 3120 DRM_WAKEUP(&rdev->irq.vblank_queue); 3121 } 3122 if (atomic_read(&rdev->irq.pflip[0])) 3123 radeon_crtc_handle_flip(rdev, 0); 3124 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 3125 DRM_DEBUG("IH: D1 vblank\n"); 3126 } 3127 break; 3128 case 1: /* D1 vline */ 3129 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 3130 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 3131 DRM_DEBUG("IH: D1 vline\n"); 3132 } 3133 break; 3134 default: 3135 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3136 break; 3137 } 3138 break; 3139 case 2: /* D2 vblank/vline */ 3140 switch (src_data) { 3141 case 0: /* D2 vblank */ 3142 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 3143 if (rdev->irq.crtc_vblank_int[1]) { 3144 drm_handle_vblank(rdev->ddev, 1); 3145 rdev->pm.vblank_sync = true; 3146 DRM_WAKEUP(&rdev->irq.vblank_queue); 3147 } 3148 if (atomic_read(&rdev->irq.pflip[1])) 3149 radeon_crtc_handle_flip(rdev, 1); 3150 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 3151 DRM_DEBUG("IH: D2 vblank\n"); 3152 } 3153 break; 3154 case 1: /* D2 vline */ 3155 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 3156 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 3157 DRM_DEBUG("IH: D2 vline\n"); 3158 } 3159 break; 3160 default: 3161 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3162 break; 3163 } 3164 break; 3165 case 3: /* D3 vblank/vline */ 3166 switch (src_data) { 3167 case 0: /* D3 vblank */ 3168 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 3169 if (rdev->irq.crtc_vblank_int[2]) { 3170 drm_handle_vblank(rdev->ddev, 2); 3171 rdev->pm.vblank_sync = true; 3172 DRM_WAKEUP(&rdev->irq.vblank_queue); 3173 } 3174 if (atomic_read(&rdev->irq.pflip[2])) 3175 radeon_crtc_handle_flip(rdev, 2); 3176 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 3177 DRM_DEBUG("IH: D3 vblank\n"); 3178 } 3179 break; 3180 case 1: /* D3 vline */ 3181 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 3182 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 3183 DRM_DEBUG("IH: D3 vline\n"); 3184 } 3185 break; 3186 default: 3187 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3188 break; 3189 } 3190 break; 3191 case 4: /* D4 vblank/vline */ 3192 switch (src_data) { 3193 case 0: /* D4 vblank */ 3194 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 3195 if (rdev->irq.crtc_vblank_int[3]) { 3196 drm_handle_vblank(rdev->ddev, 3); 3197 rdev->pm.vblank_sync = true; 3198 DRM_WAKEUP(&rdev->irq.vblank_queue); 3199 } 3200 if (atomic_read(&rdev->irq.pflip[3])) 3201 radeon_crtc_handle_flip(rdev, 3); 3202 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 3203 DRM_DEBUG("IH: D4 vblank\n"); 3204 } 3205 break; 3206 case 1: /* D4 vline */ 3207 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 3208 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 3209 DRM_DEBUG("IH: D4 vline\n"); 3210 } 3211 break; 3212 default: 3213 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3214 break; 3215 } 3216 break; 3217 case 5: /* D5 vblank/vline */ 3218 switch (src_data) { 3219 case 0: /* D5 vblank */ 3220 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 3221 if (rdev->irq.crtc_vblank_int[4]) { 3222 drm_handle_vblank(rdev->ddev, 4); 3223 rdev->pm.vblank_sync = true; 3224 DRM_WAKEUP(&rdev->irq.vblank_queue); 3225 } 3226 if (atomic_read(&rdev->irq.pflip[4])) 3227 radeon_crtc_handle_flip(rdev, 4); 3228 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 3229 DRM_DEBUG("IH: D5 vblank\n"); 3230 } 3231 break; 3232 case 1: /* D5 vline */ 3233 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 3234 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 3235 DRM_DEBUG("IH: D5 vline\n"); 3236 } 3237 break; 3238 default: 3239 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3240 break; 3241 } 3242 break; 3243 case 6: /* D6 vblank/vline */ 3244 switch (src_data) { 3245 case 0: /* D6 vblank */ 3246 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 3247 if (rdev->irq.crtc_vblank_int[5]) { 3248 drm_handle_vblank(rdev->ddev, 5); 3249 rdev->pm.vblank_sync = true; 3250 DRM_WAKEUP(&rdev->irq.vblank_queue); 3251 } 3252 if (atomic_read(&rdev->irq.pflip[5])) 3253 radeon_crtc_handle_flip(rdev, 5); 3254 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 3255 DRM_DEBUG("IH: D6 vblank\n"); 3256 } 3257 break; 3258 case 1: /* D6 vline */ 3259 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 3260 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 3261 DRM_DEBUG("IH: D6 vline\n"); 3262 } 3263 break; 3264 default: 3265 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3266 break; 3267 } 3268 break; 3269 case 42: /* HPD hotplug */ 3270 switch (src_data) { 3271 case 0: 3272 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 3273 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 3274 queue_hotplug = true; 3275 DRM_DEBUG("IH: HPD1\n"); 3276 } 3277 break; 3278 case 1: 3279 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 3280 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 3281 queue_hotplug = true; 3282 DRM_DEBUG("IH: HPD2\n"); 3283 } 3284 break; 3285 case 2: 3286 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 3287 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 3288 queue_hotplug = true; 3289 DRM_DEBUG("IH: HPD3\n"); 3290 } 3291 break; 3292 case 3: 3293 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 3294 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 3295 queue_hotplug = true; 3296 DRM_DEBUG("IH: HPD4\n"); 3297 } 3298 break; 3299 case 4: 3300 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 3301 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 3302 queue_hotplug = true; 3303 DRM_DEBUG("IH: HPD5\n"); 3304 } 3305 break; 3306 case 5: 3307 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 3308 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 3309 queue_hotplug = true; 3310 DRM_DEBUG("IH: HPD6\n"); 3311 } 3312 break; 3313 default: 3314 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3315 break; 3316 } 3317 break; 3318 case 44: /* hdmi */ 3319 switch (src_data) { 3320 case 0: 3321 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 3322 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG; 3323 queue_hdmi = true; 3324 DRM_DEBUG("IH: HDMI0\n"); 3325 } 3326 break; 3327 case 1: 3328 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 3329 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG; 3330 queue_hdmi = true; 3331 DRM_DEBUG("IH: HDMI1\n"); 3332 } 3333 break; 3334 case 2: 3335 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 3336 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG; 3337 queue_hdmi = true; 3338 DRM_DEBUG("IH: HDMI2\n"); 3339 } 3340 break; 3341 case 3: 3342 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 3343 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG; 3344 queue_hdmi = true; 3345 DRM_DEBUG("IH: HDMI3\n"); 3346 } 3347 break; 3348 case 4: 3349 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 3350 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG; 3351 queue_hdmi = true; 3352 DRM_DEBUG("IH: HDMI4\n"); 3353 } 3354 break; 3355 case 5: 3356 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 3357 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG; 3358 queue_hdmi = true; 3359 DRM_DEBUG("IH: HDMI5\n"); 3360 } 3361 break; 3362 default: 3363 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 3364 break; 3365 } 3366 break; 3367 case 146: 3368 case 147: 3369 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 3370 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 3371 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 3372 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 3373 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 3374 /* reset addr and status */ 3375 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 3376 break; 3377 case 176: /* CP_INT in ring buffer */ 3378 case 177: /* CP_INT in IB1 */ 3379 case 178: /* CP_INT in IB2 */ 3380 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 3381 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 3382 break; 3383 case 181: /* CP EOP event */ 3384 DRM_DEBUG("IH: CP EOP\n"); 3385 if (rdev->family >= CHIP_CAYMAN) { 3386 switch (src_data) { 3387 case 0: 3388 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 3389 break; 3390 case 1: 3391 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 3392 break; 3393 case 2: 3394 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 3395 break; 3396 } 3397 } else 3398 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 3399 break; 3400 case 224: /* DMA trap event */ 3401 DRM_DEBUG("IH: DMA trap\n"); 3402 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 3403 break; 3404 case 233: /* GUI IDLE */ 3405 DRM_DEBUG("IH: GUI idle\n"); 3406 break; 3407 case 244: /* DMA trap event */ 3408 if (rdev->family >= CHIP_CAYMAN) { 3409 DRM_DEBUG("IH: DMA1 trap\n"); 3410 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 3411 } 3412 break; 3413 default: 3414 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3415 break; 3416 } 3417 3418 /* wptr/rptr are in bytes! */ 3419 rptr += 16; 3420 rptr &= rdev->ih.ptr_mask; 3421 } 3422 if (queue_hotplug) 3423 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work); 3424 if (queue_hdmi) 3425 taskqueue_enqueue(rdev->tq, &rdev->audio_work); 3426 rdev->ih.rptr = rptr; 3427 WREG32(IH_RB_RPTR, rdev->ih.rptr); 3428 atomic_set(&rdev->ih.lock, 0); 3429 3430 /* make sure wptr hasn't changed while processing */ 3431 wptr = evergreen_get_ih_wptr(rdev); 3432 if (wptr != rptr) 3433 goto restart_ih; 3434 3435 return IRQ_HANDLED; 3436 } 3437 3438 /** 3439 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring 3440 * 3441 * @rdev: radeon_device pointer 3442 * @fence: radeon fence object 3443 * 3444 * Add a DMA fence packet to the ring to write 3445 * the fence seq number and DMA trap packet to generate 3446 * an interrupt if needed (evergreen-SI). 3447 */ 3448 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev, 3449 struct radeon_fence *fence) 3450 { 3451 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3452 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3453 /* write the fence */ 3454 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0)); 3455 radeon_ring_write(ring, addr & 0xfffffffc); 3456 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff)); 3457 radeon_ring_write(ring, fence->seq); 3458 /* generate an interrupt */ 3459 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0)); 3460 /* flush HDP */ 3461 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0)); 3462 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); 3463 radeon_ring_write(ring, 1); 3464 } 3465 3466 /** 3467 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine 3468 * 3469 * @rdev: radeon_device pointer 3470 * @ib: IB object to schedule 3471 * 3472 * Schedule an IB in the DMA ring (evergreen). 3473 */ 3474 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev, 3475 struct radeon_ib *ib) 3476 { 3477 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3478 3479 if (rdev->wb.enabled) { 3480 u32 next_rptr = ring->wptr + 4; 3481 while ((next_rptr & 7) != 5) 3482 next_rptr++; 3483 next_rptr += 3; 3484 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1)); 3485 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3486 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); 3487 radeon_ring_write(ring, next_rptr); 3488 } 3489 3490 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. 3491 * Pad as necessary with NOPs. 3492 */ 3493 while ((ring->wptr & 7) != 5) 3494 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 3495 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0)); 3496 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); 3497 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); 3498 3499 } 3500 3501 /** 3502 * evergreen_copy_dma - copy pages using the DMA engine 3503 * 3504 * @rdev: radeon_device pointer 3505 * @src_offset: src GPU address 3506 * @dst_offset: dst GPU address 3507 * @num_gpu_pages: number of GPU pages to xfer 3508 * @fence: radeon fence object 3509 * 3510 * Copy GPU paging using the DMA engine (evergreen-cayman). 3511 * Used by the radeon ttm implementation to move pages if 3512 * registered as the asic copy callback. 3513 */ 3514 int evergreen_copy_dma(struct radeon_device *rdev, 3515 uint64_t src_offset, uint64_t dst_offset, 3516 unsigned num_gpu_pages, 3517 struct radeon_fence **fence) 3518 { 3519 struct radeon_semaphore *sem = NULL; 3520 int ring_index = rdev->asic->copy.dma_ring_index; 3521 struct radeon_ring *ring = &rdev->ring[ring_index]; 3522 u32 size_in_dw, cur_size_in_dw; 3523 int i, num_loops; 3524 int r = 0; 3525 3526 r = radeon_semaphore_create(rdev, &sem); 3527 if (r) { 3528 DRM_ERROR("radeon: moving bo (%d).\n", r); 3529 return r; 3530 } 3531 3532 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; 3533 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff); 3534 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); 3535 if (r) { 3536 DRM_ERROR("radeon: moving bo (%d).\n", r); 3537 radeon_semaphore_free(rdev, &sem, NULL); 3538 return r; 3539 } 3540 3541 if (radeon_fence_need_sync(*fence, ring->idx)) { 3542 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, 3543 ring->idx); 3544 radeon_fence_note_sync(*fence, ring->idx); 3545 } else { 3546 radeon_semaphore_free(rdev, &sem, NULL); 3547 } 3548 3549 for (i = 0; i < num_loops; i++) { 3550 cur_size_in_dw = size_in_dw; 3551 if (cur_size_in_dw > 0xFFFFF) 3552 cur_size_in_dw = 0xFFFFF; 3553 size_in_dw -= cur_size_in_dw; 3554 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw)); 3555 radeon_ring_write(ring, dst_offset & 0xfffffffc); 3556 radeon_ring_write(ring, src_offset & 0xfffffffc); 3557 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); 3558 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); 3559 src_offset += cur_size_in_dw * 4; 3560 dst_offset += cur_size_in_dw * 4; 3561 } 3562 3563 r = radeon_fence_emit(rdev, fence, ring->idx); 3564 if (r) { 3565 radeon_ring_unlock_undo(rdev, ring); 3566 return r; 3567 } 3568 3569 radeon_ring_unlock_commit(rdev, ring); 3570 radeon_semaphore_free(rdev, &sem, *fence); 3571 3572 return r; 3573 } 3574 3575 static int evergreen_startup(struct radeon_device *rdev) 3576 { 3577 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3578 int r; 3579 3580 /* enable pcie gen2 link */ 3581 evergreen_pcie_gen2_enable(rdev); 3582 3583 if (ASIC_IS_DCE5(rdev)) { 3584 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 3585 r = ni_init_microcode(rdev); 3586 if (r) { 3587 DRM_ERROR("Failed to load firmware!\n"); 3588 return r; 3589 } 3590 } 3591 r = ni_mc_load_microcode(rdev); 3592 if (r) { 3593 DRM_ERROR("Failed to load MC firmware!\n"); 3594 return r; 3595 } 3596 } else { 3597 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 3598 r = r600_init_microcode(rdev); 3599 if (r) { 3600 DRM_ERROR("Failed to load firmware!\n"); 3601 return r; 3602 } 3603 } 3604 } 3605 3606 r = r600_vram_scratch_init(rdev); 3607 if (r) 3608 return r; 3609 3610 evergreen_mc_program(rdev); 3611 if (rdev->flags & RADEON_IS_AGP) { 3612 evergreen_agp_enable(rdev); 3613 } else { 3614 r = evergreen_pcie_gart_enable(rdev); 3615 if (r) 3616 return r; 3617 } 3618 evergreen_gpu_init(rdev); 3619 3620 r = evergreen_blit_init(rdev); 3621 if (r) { 3622 r600_blit_fini(rdev); 3623 rdev->asic->copy.copy = NULL; 3624 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); 3625 } 3626 3627 /* allocate wb buffer */ 3628 r = radeon_wb_init(rdev); 3629 if (r) 3630 return r; 3631 3632 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 3633 if (r) { 3634 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 3635 return r; 3636 } 3637 3638 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 3639 if (r) { 3640 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 3641 return r; 3642 } 3643 3644 /* Enable IRQ */ 3645 r = r600_irq_init(rdev); 3646 if (r) { 3647 DRM_ERROR("radeon: IH init failed (%d).\n", r); 3648 radeon_irq_kms_fini(rdev); 3649 return r; 3650 } 3651 evergreen_irq_set(rdev); 3652 3653 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 3654 R600_CP_RB_RPTR, R600_CP_RB_WPTR, 3655 0, 0xfffff, RADEON_CP_PACKET2); 3656 if (r) 3657 return r; 3658 3659 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 3660 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 3661 DMA_RB_RPTR, DMA_RB_WPTR, 3662 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 3663 if (r) 3664 return r; 3665 3666 r = evergreen_cp_load_microcode(rdev); 3667 if (r) 3668 return r; 3669 r = evergreen_cp_resume(rdev); 3670 if (r) 3671 return r; 3672 r = r600_dma_resume(rdev); 3673 if (r) 3674 return r; 3675 3676 r = radeon_ib_pool_init(rdev); 3677 if (r) { 3678 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 3679 return r; 3680 } 3681 3682 r = r600_audio_init(rdev); 3683 if (r) { 3684 DRM_ERROR("radeon: audio init failed\n"); 3685 return r; 3686 } 3687 3688 return 0; 3689 } 3690 3691 int evergreen_resume(struct radeon_device *rdev) 3692 { 3693 int r; 3694 3695 /* reset the asic, the gfx blocks are often in a bad state 3696 * after the driver is unloaded or after a resume 3697 */ 3698 if (radeon_asic_reset(rdev)) 3699 dev_warn(rdev->dev, "GPU reset failed !\n"); 3700 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 3701 * posting will perform necessary task to bring back GPU into good 3702 * shape. 3703 */ 3704 /* post card */ 3705 atom_asic_init(rdev->mode_info.atom_context); 3706 3707 rdev->accel_working = true; 3708 r = evergreen_startup(rdev); 3709 if (r) { 3710 DRM_ERROR("evergreen startup failed on resume\n"); 3711 rdev->accel_working = false; 3712 return r; 3713 } 3714 3715 return r; 3716 3717 } 3718 3719 int evergreen_suspend(struct radeon_device *rdev) 3720 { 3721 r600_audio_fini(rdev); 3722 r700_cp_stop(rdev); 3723 r600_dma_stop(rdev); 3724 evergreen_irq_suspend(rdev); 3725 radeon_wb_disable(rdev); 3726 evergreen_pcie_gart_disable(rdev); 3727 3728 return 0; 3729 } 3730 3731 /* Plan is to move initialization in that function and use 3732 * helper function so that radeon_device_init pretty much 3733 * do nothing more than calling asic specific function. This 3734 * should also allow to remove a bunch of callback function 3735 * like vram_info. 3736 */ 3737 int evergreen_init(struct radeon_device *rdev) 3738 { 3739 int r; 3740 3741 /* Read BIOS */ 3742 if (!radeon_get_bios(rdev)) { 3743 if (ASIC_IS_AVIVO(rdev)) 3744 return -EINVAL; 3745 } 3746 /* Must be an ATOMBIOS */ 3747 if (!rdev->is_atom_bios) { 3748 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 3749 return -EINVAL; 3750 } 3751 r = radeon_atombios_init(rdev); 3752 if (r) 3753 return r; 3754 /* reset the asic, the gfx blocks are often in a bad state 3755 * after the driver is unloaded or after a resume 3756 */ 3757 if (radeon_asic_reset(rdev)) 3758 dev_warn(rdev->dev, "GPU reset failed !\n"); 3759 /* Post card if necessary */ 3760 if (!radeon_card_posted(rdev)) { 3761 if (!rdev->bios) { 3762 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 3763 return -EINVAL; 3764 } 3765 DRM_INFO("GPU not posted. posting now...\n"); 3766 atom_asic_init(rdev->mode_info.atom_context); 3767 } 3768 /* Initialize scratch registers */ 3769 r600_scratch_init(rdev); 3770 /* Initialize surface registers */ 3771 radeon_surface_init(rdev); 3772 /* Initialize clocks */ 3773 radeon_get_clock_info(rdev->ddev); 3774 /* Fence driver */ 3775 r = radeon_fence_driver_init(rdev); 3776 if (r) 3777 return r; 3778 /* initialize AGP */ 3779 if (rdev->flags & RADEON_IS_AGP) { 3780 r = radeon_agp_init(rdev); 3781 if (r) 3782 radeon_agp_disable(rdev); 3783 } 3784 /* initialize memory controller */ 3785 r = evergreen_mc_init(rdev); 3786 if (r) 3787 return r; 3788 /* Memory manager */ 3789 r = radeon_bo_init(rdev); 3790 if (r) 3791 return r; 3792 3793 r = radeon_irq_kms_init(rdev); 3794 if (r) 3795 return r; 3796 3797 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; 3798 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); 3799 3800 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; 3801 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); 3802 3803 rdev->ih.ring_obj = NULL; 3804 r600_ih_ring_init(rdev, 64 * 1024); 3805 3806 r = r600_pcie_gart_init(rdev); 3807 if (r) 3808 return r; 3809 3810 rdev->accel_working = true; 3811 r = evergreen_startup(rdev); 3812 if (r) { 3813 dev_err(rdev->dev, "disabling GPU acceleration\n"); 3814 r700_cp_fini(rdev); 3815 r600_dma_fini(rdev); 3816 r600_irq_fini(rdev); 3817 radeon_wb_fini(rdev); 3818 radeon_ib_pool_fini(rdev); 3819 radeon_irq_kms_fini(rdev); 3820 evergreen_pcie_gart_fini(rdev); 3821 rdev->accel_working = false; 3822 } 3823 3824 /* Don't start up if the MC ucode is missing on BTC parts. 3825 * The default clocks and voltages before the MC ucode 3826 * is loaded are not suffient for advanced operations. 3827 */ 3828 if (ASIC_IS_DCE5(rdev)) { 3829 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 3830 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 3831 return -EINVAL; 3832 } 3833 } 3834 3835 return 0; 3836 } 3837 3838 void evergreen_fini(struct radeon_device *rdev) 3839 { 3840 r600_audio_fini(rdev); 3841 r600_blit_fini(rdev); 3842 r700_cp_fini(rdev); 3843 r600_dma_fini(rdev); 3844 r600_irq_fini(rdev); 3845 radeon_wb_fini(rdev); 3846 radeon_ib_pool_fini(rdev); 3847 radeon_irq_kms_fini(rdev); 3848 evergreen_pcie_gart_fini(rdev); 3849 r600_vram_scratch_fini(rdev); 3850 radeon_gem_fini(rdev); 3851 radeon_fence_driver_fini(rdev); 3852 radeon_agp_fini(rdev); 3853 radeon_bo_fini(rdev); 3854 radeon_atombios_fini(rdev); 3855 if (ASIC_IS_DCE5(rdev)) 3856 ni_fini_microcode(rdev); 3857 else 3858 r600_fini_microcode(rdev); 3859 drm_free(rdev->bios, M_DRM); 3860 rdev->bios = NULL; 3861 } 3862 3863 void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 3864 { 3865 u32 link_width_cntl, speed_cntl, mask; 3866 int ret; 3867 3868 if (radeon_pcie_gen2 == 0) 3869 return; 3870 3871 if (rdev->flags & RADEON_IS_IGP) 3872 return; 3873 3874 if (!(rdev->flags & RADEON_IS_PCIE)) 3875 return; 3876 3877 /* x2 cards have a special sequence */ 3878 if (ASIC_IS_X2(rdev)) 3879 return; 3880 3881 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 3882 if (ret != 0) 3883 return; 3884 3885 if (!(mask & DRM_PCIE_SPEED_50)) 3886 return; 3887 3888 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3889 if (speed_cntl & LC_CURRENT_DATA_RATE) { 3890 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 3891 return; 3892 } 3893 3894 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 3895 3896 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 3897 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 3898 3899 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); 3900 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 3901 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 3902 3903 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3904 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 3905 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3906 3907 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3908 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 3909 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3910 3911 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3912 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 3913 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3914 3915 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3916 speed_cntl |= LC_GEN2_EN_STRAP; 3917 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3918 3919 } else { 3920 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); 3921 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 3922 if (1) 3923 link_width_cntl |= LC_UPCONFIGURE_DIS; 3924 else 3925 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 3926 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 3927 } 3928 } 3929