drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/slab.h>
  29 #include <linux/seq_file.h>
  30 #include <linux/firmware.h>
  31 #include <linux/platform_device.h>
  32 #include "drmP.h"
  33 #include "radeon_drm.h"
  34 #include "radeon.h"
  35 #include "radeon_asic.h"
  36 #include "radeon_mode.h"
  37 #include "r600d.h"
  38 #include "atom.h"
  39 #include "avivod.h"
  40
  41 #define PFP_UCODE_SIZE 576
  42 #define PM4_UCODE_SIZE 1792
  43 #define RLC_UCODE_SIZE 768
  44 #define R700_PFP_UCODE_SIZE 848
  45 #define R700_PM4_UCODE_SIZE 1360
  46 #define R700_RLC_UCODE_SIZE 1024
  47 #define EVERGREEN_PFP_UCODE_SIZE 1120
  48 #define EVERGREEN_PM4_UCODE_SIZE 1376
  49 #define EVERGREEN_RLC_UCODE_SIZE 768
  50
  51 /* Firmware Names */
  52 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  53 MODULE_FIRMWARE("radeon/R600_me.bin");
  54 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  55 MODULE_FIRMWARE("radeon/RV610_me.bin");
  56 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  57 MODULE_FIRMWARE("radeon/RV630_me.bin");
  58 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  59 MODULE_FIRMWARE("radeon/RV620_me.bin");
  60 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  61 MODULE_FIRMWARE("radeon/RV635_me.bin");
  62 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  63 MODULE_FIRMWARE("radeon/RV670_me.bin");
  64 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  65 MODULE_FIRMWARE("radeon/RS780_me.bin");
  66 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  67 MODULE_FIRMWARE("radeon/RV770_me.bin");
  68 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  69 MODULE_FIRMWARE("radeon/RV730_me.bin");
  70 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  71 MODULE_FIRMWARE("radeon/RV710_me.bin");
  72 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  73 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  74 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  75 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  76 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
  77 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  78 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  79 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
  80 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  81 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  82 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
  83 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
  84 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  85 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
  86
  87 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
  88
  89 /* r600,rv610,rv630,rv620,rv635,rv670 */
  90 int r600_mc_wait_for_idle(struct radeon_device *rdev);
  91 void r600_gpu_init(struct radeon_device *rdev);
  92 void r600_fini(struct radeon_device *rdev);
  93 void r600_irq_disable(struct radeon_device *rdev);
  94
  95 void r600_set_power_state(struct radeon_device *rdev)
  96 {
  97         /* if *_clock_mode are the same, *_power_state are as well */
  98         if (rdev->pm.requested_clock_mode == rdev->pm.current_clock_mode)
  99                 return;
 100
 101         DRM_INFO("Setting: e: %d m: %d p: %d\n",
 102                  rdev->pm.requested_clock_mode->sclk,
 103                  rdev->pm.requested_clock_mode->mclk,
 104                  rdev->pm.requested_power_state->non_clock_info.pcie_lanes);
 105
 106         /* set pcie lanes */
 107         /* TODO */
 108
 109         /* set voltage */
 110         /* TODO */
 111
 112         /* set engine clock */
 113         radeon_sync_with_vblank(rdev);
 114         radeon_pm_debug_check_in_vbl(rdev, false);
 115         radeon_set_engine_clock(rdev, rdev->pm.requested_clock_mode->sclk);
 116         radeon_pm_debug_check_in_vbl(rdev, true);
 117
 118 #if 0
 119         /* set memory clock */
 120         if (rdev->asic->set_memory_clock) {
 121                 radeon_sync_with_vblank(rdev);
 122                 radeon_pm_debug_check_in_vbl(rdev, false);
 123                 radeon_set_memory_clock(rdev, rdev->pm.requested_clock_mode->mclk);
 124                 radeon_pm_debug_check_in_vbl(rdev, true);
 125         }
 126 #endif
 127
 128         rdev->pm.current_power_state = rdev->pm.requested_power_state;
 129         rdev->pm.current_clock_mode = rdev->pm.requested_clock_mode;
 130 }
 131
 132 bool r600_gui_idle(struct radeon_device *rdev)
 133 {
 134         if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
 135                 return false;
 136         else
 137                 return true;
 138 }
 139
 140 /* hpd for digital panel detect/disconnect */
 141 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 142 {
 143         bool connected = false;
 144
 145         if (ASIC_IS_DCE3(rdev)) {
 146                 switch (hpd) {
 147                 case RADEON_HPD_1:
 148                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 149                                 connected = true;
 150                         break;
 151                 case RADEON_HPD_2:
 152                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 153                                 connected = true;
 154                         break;
 155                 case RADEON_HPD_3:
 156                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 157                                 connected = true;
 158                         break;
 159                 case RADEON_HPD_4:
 160                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 161                                 connected = true;
 162                         break;
 163                         /* DCE 3.2 */
 164                 case RADEON_HPD_5:
 165                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 166                                 connected = true;
 167                         break;
 168                 case RADEON_HPD_6:
 169                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 170                                 connected = true;
 171                         break;
 172                 default:
 173                         break;
 174                 }
 175         } else {
 176                 switch (hpd) {
 177                 case RADEON_HPD_1:
 178                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 179                                 connected = true;
 180                         break;
 181                 case RADEON_HPD_2:
 182                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 183                                 connected = true;
 184                         break;
 185                 case RADEON_HPD_3:
 186                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 187                                 connected = true;
 188                         break;
 189                 default:
 190                         break;
 191                 }
 192         }
 193         return connected;
 194 }
 195
 196 void r600_hpd_set_polarity(struct radeon_device *rdev,
 197                            enum radeon_hpd_id hpd)
 198 {
 199         u32 tmp;
 200         bool connected = r600_hpd_sense(rdev, hpd);
 201
 202         if (ASIC_IS_DCE3(rdev)) {
 203                 switch (hpd) {
 204                 case RADEON_HPD_1:
 205                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 206                         if (connected)
 207                                 tmp &= ~DC_HPDx_INT_POLARITY;
 208                         else
 209                                 tmp |= DC_HPDx_INT_POLARITY;
 210                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 211                         break;
 212                 case RADEON_HPD_2:
 213                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 214                         if (connected)
 215                                 tmp &= ~DC_HPDx_INT_POLARITY;
 216                         else
 217                                 tmp |= DC_HPDx_INT_POLARITY;
 218                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 219                         break;
 220                 case RADEON_HPD_3:
 221                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 222                         if (connected)
 223                                 tmp &= ~DC_HPDx_INT_POLARITY;
 224                         else
 225                                 tmp |= DC_HPDx_INT_POLARITY;
 226                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 227                         break;
 228                 case RADEON_HPD_4:
 229                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 230                         if (connected)
 231                                 tmp &= ~DC_HPDx_INT_POLARITY;
 232                         else
 233                                 tmp |= DC_HPDx_INT_POLARITY;
 234                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 235                         break;
 236                 case RADEON_HPD_5:
 237                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 238                         if (connected)
 239                                 tmp &= ~DC_HPDx_INT_POLARITY;
 240                         else
 241                                 tmp |= DC_HPDx_INT_POLARITY;
 242                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 243                         break;
 244                         /* DCE 3.2 */
 245                 case RADEON_HPD_6:
 246                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 247                         if (connected)
 248                                 tmp &= ~DC_HPDx_INT_POLARITY;
 249                         else
 250                                 tmp |= DC_HPDx_INT_POLARITY;
 251                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 252                         break;
 253                 default:
 254                         break;
 255                 }
 256         } else {
 257                 switch (hpd) {
 258                 case RADEON_HPD_1:
 259                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 260                         if (connected)
 261                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 262                         else
 263                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 264                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 265                         break;
 266                 case RADEON_HPD_2:
 267                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 268                         if (connected)
 269                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 270                         else
 271                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 272                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 273                         break;
 274                 case RADEON_HPD_3:
 275                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 276                         if (connected)
 277                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 278                         else
 279                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 280                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 281                         break;
 282                 default:
 283                         break;
 284                 }
 285         }
 286 }
 287
 288 void r600_hpd_init(struct radeon_device *rdev)
 289 {
 290         struct drm_device *dev = rdev->ddev;
 291         struct drm_connector *connector;
 292
 293         if (ASIC_IS_DCE3(rdev)) {
 294                 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 295                 if (ASIC_IS_DCE32(rdev))
 296                         tmp |= DC_HPDx_EN;
 297
 298                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 299                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 300                         switch (radeon_connector->hpd.hpd) {
 301                         case RADEON_HPD_1:
 302                                 WREG32(DC_HPD1_CONTROL, tmp);
 303                                 rdev->irq.hpd[0] = true;
 304                                 break;
 305                         case RADEON_HPD_2:
 306                                 WREG32(DC_HPD2_CONTROL, tmp);
 307                                 rdev->irq.hpd[1] = true;
 308                                 break;
 309                         case RADEON_HPD_3:
 310                                 WREG32(DC_HPD3_CONTROL, tmp);
 311                                 rdev->irq.hpd[2] = true;
 312                                 break;
 313                         case RADEON_HPD_4:
 314                                 WREG32(DC_HPD4_CONTROL, tmp);
 315                                 rdev->irq.hpd[3] = true;
 316                                 break;
 317                                 /* DCE 3.2 */
 318                         case RADEON_HPD_5:
 319                                 WREG32(DC_HPD5_CONTROL, tmp);
 320                                 rdev->irq.hpd[4] = true;
 321                                 break;
 322                         case RADEON_HPD_6:
 323                                 WREG32(DC_HPD6_CONTROL, tmp);
 324                                 rdev->irq.hpd[5] = true;
 325                                 break;
 326                         default:
 327                                 break;
 328                         }
 329                 }
 330         } else {
 331                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 332                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 333                         switch (radeon_connector->hpd.hpd) {
 334                         case RADEON_HPD_1:
 335                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 336                                 rdev->irq.hpd[0] = true;
 337                                 break;
 338                         case RADEON_HPD_2:
 339                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 340                                 rdev->irq.hpd[1] = true;
 341                                 break;
 342                         case RADEON_HPD_3:
 343                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 344                                 rdev->irq.hpd[2] = true;
 345                                 break;
 346                         default:
 347                                 break;
 348                         }
 349                 }
 350         }
 351         if (rdev->irq.installed)
 352                 r600_irq_set(rdev);
 353 }
 354
 355 void r600_hpd_fini(struct radeon_device *rdev)
 356 {
 357         struct drm_device *dev = rdev->ddev;
 358         struct drm_connector *connector;
 359
 360         if (ASIC_IS_DCE3(rdev)) {
 361                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 362                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 363                         switch (radeon_connector->hpd.hpd) {
 364                         case RADEON_HPD_1:
 365                                 WREG32(DC_HPD1_CONTROL, 0);
 366                                 rdev->irq.hpd[0] = false;
 367                                 break;
 368                         case RADEON_HPD_2:
 369                                 WREG32(DC_HPD2_CONTROL, 0);
 370                                 rdev->irq.hpd[1] = false;
 371                                 break;
 372                         case RADEON_HPD_3:
 373                                 WREG32(DC_HPD3_CONTROL, 0);
 374                                 rdev->irq.hpd[2] = false;
 375                                 break;
 376                         case RADEON_HPD_4:
 377                                 WREG32(DC_HPD4_CONTROL, 0);
 378                                 rdev->irq.hpd[3] = false;
 379                                 break;
 380                                 /* DCE 3.2 */
 381                         case RADEON_HPD_5:
 382                                 WREG32(DC_HPD5_CONTROL, 0);
 383                                 rdev->irq.hpd[4] = false;
 384                                 break;
 385                         case RADEON_HPD_6:
 386                                 WREG32(DC_HPD6_CONTROL, 0);
 387                                 rdev->irq.hpd[5] = false;
 388                                 break;
 389                         default:
 390                                 break;
 391                         }
 392                 }
 393         } else {
 394                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 395                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 396                         switch (radeon_connector->hpd.hpd) {
 397                         case RADEON_HPD_1:
 398                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 399                                 rdev->irq.hpd[0] = false;
 400                                 break;
 401                         case RADEON_HPD_2:
 402                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 403                                 rdev->irq.hpd[1] = false;
 404                                 break;
 405                         case RADEON_HPD_3:
 406                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 407                                 rdev->irq.hpd[2] = false;
 408                                 break;
 409                         default:
 410                                 break;
 411                         }
 412                 }
 413         }
 414 }
 415
 416 /*
 417  * R600 PCIE GART
 418  */
 419 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 420 {
 421         unsigned i;
 422         u32 tmp;
 423
 424         /* flush hdp cache so updates hit vram */
 425         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 426
 427         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 428         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 429         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 430         for (i = 0; i < rdev->usec_timeout; i++) {
 431                 /* read MC_STATUS */
 432                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 433                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 434                 if (tmp == 2) {
 435                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 436                         return;
 437                 }
 438                 if (tmp) {
 439                         return;
 440                 }
 441                 udelay(1);
 442         }
 443 }
 444
 445 int r600_pcie_gart_init(struct radeon_device *rdev)
 446 {
 447         int r;
 448
 449         if (rdev->gart.table.vram.robj) {
 450                 WARN(1, "R600 PCIE GART already initialized.\n");
 451                 return 0;
 452         }
 453         /* Initialize common gart structure */
 454         r = radeon_gart_init(rdev);
 455         if (r)
 456                 return r;
 457         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 458         return radeon_gart_table_vram_alloc(rdev);
 459 }
 460
 461 int r600_pcie_gart_enable(struct radeon_device *rdev)
 462 {
 463         u32 tmp;
 464         int r, i;
 465
 466         if (rdev->gart.table.vram.robj == NULL) {
 467                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 468                 return -EINVAL;
 469         }
 470         r = radeon_gart_table_vram_pin(rdev);
 471         if (r)
 472                 return r;
 473         radeon_gart_restore(rdev);
 474
 475         /* Setup L2 cache */
 476         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 477                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 478                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 479         WREG32(VM_L2_CNTL2, 0);
 480         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 481         /* Setup TLB control */
 482         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 483                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 484                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 485                 ENABLE_WAIT_L2_QUERY;
 486         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 487         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 488         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 489         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 490         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 491         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 492         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 493         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 494         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 495         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 496         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 497         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 498         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 499         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 500         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 501         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 502         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 503         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 504                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 505         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 506                         (u32)(rdev->dummy_page.addr >> 12));
 507         for (i = 1; i < 7; i++)
 508                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 509
 510         r600_pcie_gart_tlb_flush(rdev);
 511         rdev->gart.ready = true;
 512         return 0;
 513 }
 514
 515 void r600_pcie_gart_disable(struct radeon_device *rdev)
 516 {
 517         u32 tmp;
 518         int i, r;
 519
 520         /* Disable all tables */
 521         for (i = 0; i < 7; i++)
 522                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 523
 524         /* Disable L2 cache */
 525         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 526                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 527         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 528         /* Setup L1 TLB control */
 529         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 530                 ENABLE_WAIT_L2_QUERY;
 531         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 532         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 533         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 534         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 535         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 536         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 537         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 538         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 539         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 540         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 541         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 542         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 543         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 544         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 545         if (rdev->gart.table.vram.robj) {
 546                 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
 547                 if (likely(r == 0)) {
 548                         radeon_bo_kunmap(rdev->gart.table.vram.robj);
 549                         radeon_bo_unpin(rdev->gart.table.vram.robj);
 550                         radeon_bo_unreserve(rdev->gart.table.vram.robj);
 551                 }
 552         }
 553 }
 554
 555 void r600_pcie_gart_fini(struct radeon_device *rdev)
 556 {
 557         radeon_gart_fini(rdev);
 558         r600_pcie_gart_disable(rdev);
 559         radeon_gart_table_vram_free(rdev);
 560 }
 561
 562 void r600_agp_enable(struct radeon_device *rdev)
 563 {
 564         u32 tmp;
 565         int i;
 566
 567         /* Setup L2 cache */
 568         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 569                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 570                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 571         WREG32(VM_L2_CNTL2, 0);
 572         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 573         /* Setup TLB control */
 574         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 575                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 576                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 577                 ENABLE_WAIT_L2_QUERY;
 578         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 579         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 580         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 581         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 582         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 583         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 584         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 585         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 586         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 587         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 588         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 589         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 590         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 591         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 592         for (i = 0; i < 7; i++)
 593                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 594 }
 595
 596 int r600_mc_wait_for_idle(struct radeon_device *rdev)
 597 {
 598         unsigned i;
 599         u32 tmp;
 600
 601         for (i = 0; i < rdev->usec_timeout; i++) {
 602                 /* read MC_STATUS */
 603                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
 604                 if (!tmp)
 605                         return 0;
 606                 udelay(1);
 607         }
 608         return -1;
 609 }
 610
 611 static void r600_mc_program(struct radeon_device *rdev)
 612 {
 613         struct rv515_mc_save save;
 614         u32 tmp;
 615         int i, j;
 616
 617         /* Initialize HDP */
 618         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 619                 WREG32((0x2c14 + j), 0x00000000);
 620                 WREG32((0x2c18 + j), 0x00000000);
 621                 WREG32((0x2c1c + j), 0x00000000);
 622                 WREG32((0x2c20 + j), 0x00000000);
 623                 WREG32((0x2c24 + j), 0x00000000);
 624         }
 625         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
 626
 627         rv515_mc_stop(rdev, &save);
 628         if (r600_mc_wait_for_idle(rdev)) {
 629                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 630         }
 631         /* Lockout access through VGA aperture (doesn't exist before R600) */
 632         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
 633         /* Update configuration */
 634         if (rdev->flags & RADEON_IS_AGP) {
 635                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
 636                         /* VRAM before AGP */
 637                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 638                                 rdev->mc.vram_start >> 12);
 639                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 640                                 rdev->mc.gtt_end >> 12);
 641                 } else {
 642                         /* VRAM after AGP */
 643                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 644                                 rdev->mc.gtt_start >> 12);
 645                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 646                                 rdev->mc.vram_end >> 12);
 647                 }
 648         } else {
 649                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
 650                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
 651         }
 652         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 653         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
 654         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
 655         WREG32(MC_VM_FB_LOCATION, tmp);
 656         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
 657         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
 658         WREG32(HDP_NONSURFACE_SIZE, rdev->mc.mc_vram_size | 0x3FF);
 659         if (rdev->flags & RADEON_IS_AGP) {
 660                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
 661                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
 662                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
 663         } else {
 664                 WREG32(MC_VM_AGP_BASE, 0);
 665                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
 666                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
 667         }
 668         if (r600_mc_wait_for_idle(rdev)) {
 669                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 670         }
 671         rv515_mc_resume(rdev, &save);
 672         /* we need to own VRAM, so turn off the VGA renderer here
 673          * to stop it overwriting our objects */
 674         rv515_vga_render_disable(rdev);
 675 }
 676
 677 /**
 678  * r600_vram_gtt_location - try to find VRAM & GTT location
 679  * @rdev: radeon device structure holding all necessary informations
 680  * @mc: memory controller structure holding memory informations
 681  *
 682  * Function will place try to place VRAM at same place as in CPU (PCI)
 683  * address space as some GPU seems to have issue when we reprogram at
 684  * different address space.
 685  *
 686  * If there is not enough space to fit the unvisible VRAM after the
 687  * aperture then we limit the VRAM size to the aperture.
 688  *
 689  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
 690  * them to be in one from GPU point of view so that we can program GPU to
 691  * catch access outside them (weird GPU policy see ??).
 692  *
 693  * This function will never fails, worst case are limiting VRAM or GTT.
 694  *
 695  * Note: GTT start, end, size should be initialized before calling this
 696  * function on AGP platform.
 697  */
 698 void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 699 {
 700         u64 size_bf, size_af;
 701
 702         if (mc->mc_vram_size > 0xE0000000) {
 703                 /* leave room for at least 512M GTT */
 704                 dev_warn(rdev->dev, "limiting VRAM\n");
 705                 mc->real_vram_size = 0xE0000000;
 706                 mc->mc_vram_size = 0xE0000000;
 707         }
 708         if (rdev->flags & RADEON_IS_AGP) {
 709                 size_bf = mc->gtt_start;
 710                 size_af = 0xFFFFFFFF - mc->gtt_end + 1;
 711                 if (size_bf > size_af) {
 712                         if (mc->mc_vram_size > size_bf) {
 713                                 dev_warn(rdev->dev, "limiting VRAM\n");
 714                                 mc->real_vram_size = size_bf;
 715                                 mc->mc_vram_size = size_bf;
 716                         }
 717                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
 718                 } else {
 719                         if (mc->mc_vram_size > size_af) {
 720                                 dev_warn(rdev->dev, "limiting VRAM\n");
 721                                 mc->real_vram_size = size_af;
 722                                 mc->mc_vram_size = size_af;
 723                         }
 724                         mc->vram_start = mc->gtt_end;
 725                 }
 726                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 727                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
 728                                 mc->mc_vram_size >> 20, mc->vram_start,
 729                                 mc->vram_end, mc->real_vram_size >> 20);
 730         } else {
 731                 u64 base = 0;
 732                 if (rdev->flags & RADEON_IS_IGP)
 733                         base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
 734                 radeon_vram_location(rdev, &rdev->mc, base);
 735                 radeon_gtt_location(rdev, mc);
 736         }
 737 }
 738
 739 int r600_mc_init(struct radeon_device *rdev)
 740 {
 741         u32 tmp;
 742         int chansize, numchan;
 743
 744         /* Get VRAM informations */
 745         rdev->mc.vram_is_ddr = true;
 746         tmp = RREG32(RAMCFG);
 747         if (tmp & CHANSIZE_OVERRIDE) {
 748                 chansize = 16;
 749         } else if (tmp & CHANSIZE_MASK) {
 750                 chansize = 64;
 751         } else {
 752                 chansize = 32;
 753         }
 754         tmp = RREG32(CHMAP);
 755         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
 756         case 0:
 757         default:
 758                 numchan = 1;
 759                 break;
 760         case 1:
 761                 numchan = 2;
 762                 break;
 763         case 2:
 764                 numchan = 4;
 765                 break;
 766         case 3:
 767                 numchan = 8;
 768                 break;
 769         }
 770         rdev->mc.vram_width = numchan * chansize;
 771         /* Could aper size report 0 ? */
 772         rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
 773         rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
 774         /* Setup GPU memory space */
 775         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
 776         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
 777         rdev->mc.visible_vram_size = rdev->mc.aper_size;
 778         r600_vram_gtt_location(rdev, &rdev->mc);
 779
 780         if (rdev->flags & RADEON_IS_IGP)
 781                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
 782         radeon_update_bandwidth_info(rdev);
 783         return 0;
 784 }
 785
 786 /* We doesn't check that the GPU really needs a reset we simply do the
 787  * reset, it's up to the caller to determine if the GPU needs one. We
 788  * might add an helper function to check that.
 789  */
 790 int r600_gpu_soft_reset(struct radeon_device *rdev)
 791 {
 792         struct rv515_mc_save save;
 793         u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) |
 794                                 S_008010_VGT_BUSY(1) | S_008010_TA03_BUSY(1) |
 795                                 S_008010_TC_BUSY(1) | S_008010_SX_BUSY(1) |
 796                                 S_008010_SH_BUSY(1) | S_008010_SPI03_BUSY(1) |
 797                                 S_008010_SMX_BUSY(1) | S_008010_SC_BUSY(1) |
 798                                 S_008010_PA_BUSY(1) | S_008010_DB03_BUSY(1) |
 799                                 S_008010_CR_BUSY(1) | S_008010_CB03_BUSY(1) |
 800                                 S_008010_GUI_ACTIVE(1);
 801         u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) | S_008014_SPI1_BUSY(1) |
 802                         S_008014_SPI2_BUSY(1) | S_008014_SPI3_BUSY(1) |
 803                         S_008014_TA0_BUSY(1) | S_008014_TA1_BUSY(1) |
 804                         S_008014_TA2_BUSY(1) | S_008014_TA3_BUSY(1) |
 805                         S_008014_DB0_BUSY(1) | S_008014_DB1_BUSY(1) |
 806                         S_008014_DB2_BUSY(1) | S_008014_DB3_BUSY(1) |
 807                         S_008014_CB0_BUSY(1) | S_008014_CB1_BUSY(1) |
 808                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
 809         u32 tmp;
 810
 811         dev_info(rdev->dev, "GPU softreset \n");
 812         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
 813                 RREG32(R_008010_GRBM_STATUS));
 814         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
 815                 RREG32(R_008014_GRBM_STATUS2));
 816         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
 817                 RREG32(R_000E50_SRBM_STATUS));
 818         rv515_mc_stop(rdev, &save);
 819         if (r600_mc_wait_for_idle(rdev)) {
 820                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 821         }
 822         /* Disable CP parsing/prefetching */
 823         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
 824         /* Check if any of the rendering block is busy and reset it */
 825         if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) ||
 826             (RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
 827                 tmp = S_008020_SOFT_RESET_CR(1) |
 828                         S_008020_SOFT_RESET_DB(1) |
 829                         S_008020_SOFT_RESET_CB(1) |
 830                         S_008020_SOFT_RESET_PA(1) |
 831                         S_008020_SOFT_RESET_SC(1) |
 832                         S_008020_SOFT_RESET_SMX(1) |
 833                         S_008020_SOFT_RESET_SPI(1) |
 834                         S_008020_SOFT_RESET_SX(1) |
 835                         S_008020_SOFT_RESET_SH(1) |
 836                         S_008020_SOFT_RESET_TC(1) |
 837                         S_008020_SOFT_RESET_TA(1) |
 838                         S_008020_SOFT_RESET_VC(1) |
 839                         S_008020_SOFT_RESET_VGT(1);
 840                 dev_info(rdev->dev, "  R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
 841                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
 842                 RREG32(R_008020_GRBM_SOFT_RESET);
 843                 mdelay(15);
 844                 WREG32(R_008020_GRBM_SOFT_RESET, 0);
 845         }
 846         /* Reset CP (we always reset CP) */
 847         tmp = S_008020_SOFT_RESET_CP(1);
 848         dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
 849         WREG32(R_008020_GRBM_SOFT_RESET, tmp);
 850         RREG32(R_008020_GRBM_SOFT_RESET);
 851         mdelay(15);
 852         WREG32(R_008020_GRBM_SOFT_RESET, 0);
 853         /* Wait a little for things to settle down */
 854         mdelay(1);
 855         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
 856                 RREG32(R_008010_GRBM_STATUS));
 857         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
 858                 RREG32(R_008014_GRBM_STATUS2));
 859         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
 860                 RREG32(R_000E50_SRBM_STATUS));
 861         rv515_mc_resume(rdev, &save);
 862         return 0;
 863 }
 864
 865 bool r600_gpu_is_lockup(struct radeon_device *rdev)
 866 {
 867         u32 srbm_status;
 868         u32 grbm_status;
 869         u32 grbm_status2;
 870         int r;
 871
 872         srbm_status = RREG32(R_000E50_SRBM_STATUS);
 873         grbm_status = RREG32(R_008010_GRBM_STATUS);
 874         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
 875         if (!G_008010_GUI_ACTIVE(grbm_status)) {
 876                 r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
 877                 return false;
 878         }
 879         /* force CP activities */
 880         r = radeon_ring_lock(rdev, 2);
 881         if (!r) {
 882                 /* PACKET2 NOP */
 883                 radeon_ring_write(rdev, 0x80000000);
 884                 radeon_ring_write(rdev, 0x80000000);
 885                 radeon_ring_unlock_commit(rdev);
 886         }
 887         rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
 888         return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
 889 }
 890
 891 int r600_asic_reset(struct radeon_device *rdev)
 892 {
 893         return r600_gpu_soft_reset(rdev);
 894 }
 895
 896 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
 897                                              u32 num_backends,
 898                                              u32 backend_disable_mask)
 899 {
 900         u32 backend_map = 0;
 901         u32 enabled_backends_mask;
 902         u32 enabled_backends_count;
 903         u32 cur_pipe;
 904         u32 swizzle_pipe[R6XX_MAX_PIPES];
 905         u32 cur_backend;
 906         u32 i;
 907
 908         if (num_tile_pipes > R6XX_MAX_PIPES)
 909                 num_tile_pipes = R6XX_MAX_PIPES;
 910         if (num_tile_pipes < 1)
 911                 num_tile_pipes = 1;
 912         if (num_backends > R6XX_MAX_BACKENDS)
 913                 num_backends = R6XX_MAX_BACKENDS;
 914         if (num_backends < 1)
 915                 num_backends = 1;
 916
 917         enabled_backends_mask = 0;
 918         enabled_backends_count = 0;
 919         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
 920                 if (((backend_disable_mask >> i) & 1) == 0) {
 921                         enabled_backends_mask |= (1 << i);
 922                         ++enabled_backends_count;
 923                 }
 924                 if (enabled_backends_count == num_backends)
 925                         break;
 926         }
 927
 928         if (enabled_backends_count == 0) {
 929                 enabled_backends_mask = 1;
 930                 enabled_backends_count = 1;
 931         }
 932
 933         if (enabled_backends_count != num_backends)
 934                 num_backends = enabled_backends_count;
 935
 936         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
 937         switch (num_tile_pipes) {
 938         case 1:
 939                 swizzle_pipe[0] = 0;
 940                 break;
 941         case 2:
 942                 swizzle_pipe[0] = 0;
 943                 swizzle_pipe[1] = 1;
 944                 break;
 945         case 3:
 946                 swizzle_pipe[0] = 0;
 947                 swizzle_pipe[1] = 1;
 948                 swizzle_pipe[2] = 2;
 949                 break;
 950         case 4:
 951                 swizzle_pipe[0] = 0;
 952                 swizzle_pipe[1] = 1;
 953                 swizzle_pipe[2] = 2;
 954                 swizzle_pipe[3] = 3;
 955                 break;
 956         case 5:
 957                 swizzle_pipe[0] = 0;
 958                 swizzle_pipe[1] = 1;
 959                 swizzle_pipe[2] = 2;
 960                 swizzle_pipe[3] = 3;
 961                 swizzle_pipe[4] = 4;
 962                 break;
 963         case 6:
 964                 swizzle_pipe[0] = 0;
 965                 swizzle_pipe[1] = 2;
 966                 swizzle_pipe[2] = 4;
 967                 swizzle_pipe[3] = 5;
 968                 swizzle_pipe[4] = 1;
 969                 swizzle_pipe[5] = 3;
 970                 break;
 971         case 7:
 972                 swizzle_pipe[0] = 0;
 973                 swizzle_pipe[1] = 2;
 974                 swizzle_pipe[2] = 4;
 975                 swizzle_pipe[3] = 6;
 976                 swizzle_pipe[4] = 1;
 977                 swizzle_pipe[5] = 3;
 978                 swizzle_pipe[6] = 5;
 979                 break;
 980         case 8:
 981                 swizzle_pipe[0] = 0;
 982                 swizzle_pipe[1] = 2;
 983                 swizzle_pipe[2] = 4;
 984                 swizzle_pipe[3] = 6;
 985                 swizzle_pipe[4] = 1;
 986                 swizzle_pipe[5] = 3;
 987                 swizzle_pipe[6] = 5;
 988                 swizzle_pipe[7] = 7;
 989                 break;
 990         }
 991
 992         cur_backend = 0;
 993         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
 994                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
 995                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 996
 997                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
 998
 999                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
1000         }
1001
1002         return backend_map;
1003 }
1004
1005 int r600_count_pipe_bits(uint32_t val)
1006 {
1007         int i, ret = 0;
1008
1009         for (i = 0; i < 32; i++) {
1010                 ret += val & 1;
1011                 val >>= 1;
1012         }
1013         return ret;
1014 }
1015
1016 void r600_gpu_init(struct radeon_device *rdev)
1017 {
1018         u32 tiling_config;
1019         u32 ramcfg;
1020         u32 backend_map;
1021         u32 cc_rb_backend_disable;
1022         u32 cc_gc_shader_pipe_config;
1023         u32 tmp;
1024         int i, j;
1025         u32 sq_config;
1026         u32 sq_gpr_resource_mgmt_1 = 0;
1027         u32 sq_gpr_resource_mgmt_2 = 0;
1028         u32 sq_thread_resource_mgmt = 0;
1029         u32 sq_stack_resource_mgmt_1 = 0;
1030         u32 sq_stack_resource_mgmt_2 = 0;
1031
1032         /* FIXME: implement */
1033         switch (rdev->family) {
1034         case CHIP_R600:
1035                 rdev->config.r600.max_pipes = 4;
1036                 rdev->config.r600.max_tile_pipes = 8;
1037                 rdev->config.r600.max_simds = 4;
1038                 rdev->config.r600.max_backends = 4;
1039                 rdev->config.r600.max_gprs = 256;
1040                 rdev->config.r600.max_threads = 192;
1041                 rdev->config.r600.max_stack_entries = 256;
1042                 rdev->config.r600.max_hw_contexts = 8;
1043                 rdev->config.r600.max_gs_threads = 16;
1044                 rdev->config.r600.sx_max_export_size = 128;
1045                 rdev->config.r600.sx_max_export_pos_size = 16;
1046                 rdev->config.r600.sx_max_export_smx_size = 128;
1047                 rdev->config.r600.sq_num_cf_insts = 2;
1048                 break;
1049         case CHIP_RV630:
1050         case CHIP_RV635:
1051                 rdev->config.r600.max_pipes = 2;
1052                 rdev->config.r600.max_tile_pipes = 2;
1053                 rdev->config.r600.max_simds = 3;
1054                 rdev->config.r600.max_backends = 1;
1055                 rdev->config.r600.max_gprs = 128;
1056                 rdev->config.r600.max_threads = 192;
1057                 rdev->config.r600.max_stack_entries = 128;
1058                 rdev->config.r600.max_hw_contexts = 8;
1059                 rdev->config.r600.max_gs_threads = 4;
1060                 rdev->config.r600.sx_max_export_size = 128;
1061                 rdev->config.r600.sx_max_export_pos_size = 16;
1062                 rdev->config.r600.sx_max_export_smx_size = 128;
1063                 rdev->config.r600.sq_num_cf_insts = 2;
1064                 break;
1065         case CHIP_RV610:
1066         case CHIP_RV620:
1067         case CHIP_RS780:
1068         case CHIP_RS880:
1069                 rdev->config.r600.max_pipes = 1;
1070                 rdev->config.r600.max_tile_pipes = 1;
1071                 rdev->config.r600.max_simds = 2;
1072                 rdev->config.r600.max_backends = 1;
1073                 rdev->config.r600.max_gprs = 128;
1074                 rdev->config.r600.max_threads = 192;
1075                 rdev->config.r600.max_stack_entries = 128;
1076                 rdev->config.r600.max_hw_contexts = 4;
1077                 rdev->config.r600.max_gs_threads = 4;
1078                 rdev->config.r600.sx_max_export_size = 128;
1079                 rdev->config.r600.sx_max_export_pos_size = 16;
1080                 rdev->config.r600.sx_max_export_smx_size = 128;
1081                 rdev->config.r600.sq_num_cf_insts = 1;
1082                 break;
1083         case CHIP_RV670:
1084                 rdev->config.r600.max_pipes = 4;
1085                 rdev->config.r600.max_tile_pipes = 4;
1086                 rdev->config.r600.max_simds = 4;
1087                 rdev->config.r600.max_backends = 4;
1088                 rdev->config.r600.max_gprs = 192;
1089                 rdev->config.r600.max_threads = 192;
1090                 rdev->config.r600.max_stack_entries = 256;
1091                 rdev->config.r600.max_hw_contexts = 8;
1092                 rdev->config.r600.max_gs_threads = 16;
1093                 rdev->config.r600.sx_max_export_size = 128;
1094                 rdev->config.r600.sx_max_export_pos_size = 16;
1095                 rdev->config.r600.sx_max_export_smx_size = 128;
1096                 rdev->config.r600.sq_num_cf_insts = 2;
1097                 break;
1098         default:
1099                 break;
1100         }
1101
1102         /* Initialize HDP */
1103         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1104                 WREG32((0x2c14 + j), 0x00000000);
1105                 WREG32((0x2c18 + j), 0x00000000);
1106                 WREG32((0x2c1c + j), 0x00000000);
1107                 WREG32((0x2c20 + j), 0x00000000);
1108                 WREG32((0x2c24 + j), 0x00000000);
1109         }
1110
1111         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1112
1113         /* Setup tiling */
1114         tiling_config = 0;
1115         ramcfg = RREG32(RAMCFG);
1116         switch (rdev->config.r600.max_tile_pipes) {
1117         case 1:
1118                 tiling_config |= PIPE_TILING(0);
1119                 break;
1120         case 2:
1121                 tiling_config |= PIPE_TILING(1);
1122                 break;
1123         case 4:
1124                 tiling_config |= PIPE_TILING(2);
1125                 break;
1126         case 8:
1127                 tiling_config |= PIPE_TILING(3);
1128                 break;
1129         default:
1130                 break;
1131         }
1132         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1133         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1134         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1135         tiling_config |= GROUP_SIZE(0);
1136         rdev->config.r600.tiling_group_size = 256;
1137         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1138         if (tmp > 3) {
1139                 tiling_config |= ROW_TILING(3);
1140                 tiling_config |= SAMPLE_SPLIT(3);
1141         } else {
1142                 tiling_config |= ROW_TILING(tmp);
1143                 tiling_config |= SAMPLE_SPLIT(tmp);
1144         }
1145         tiling_config |= BANK_SWAPS(1);
1146
1147         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1148         cc_rb_backend_disable |=
1149                 BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
1150
1151         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1152         cc_gc_shader_pipe_config |=
1153                 INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
1154         cc_gc_shader_pipe_config |=
1155                 INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
1156
1157         backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
1158                                                         (R6XX_MAX_BACKENDS -
1159                                                          r600_count_pipe_bits((cc_rb_backend_disable &
1160                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
1161                                                         (cc_rb_backend_disable >> 16));
1162
1163         tiling_config |= BACKEND_MAP(backend_map);
1164         WREG32(GB_TILING_CONFIG, tiling_config);
1165         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1166         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1167
1168         /* Setup pipes */
1169         WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1170         WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1171         WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1172
1173         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1174         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1175         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1176
1177         /* Setup some CP states */
1178         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1179         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1180
1181         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1182                              SYNC_WALKER | SYNC_ALIGNER));
1183         /* Setup various GPU states */
1184         if (rdev->family == CHIP_RV670)
1185                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1186
1187         tmp = RREG32(SX_DEBUG_1);
1188         tmp |= SMX_EVENT_RELEASE;
1189         if ((rdev->family > CHIP_R600))
1190                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1191         WREG32(SX_DEBUG_1, tmp);
1192
1193         if (((rdev->family) == CHIP_R600) ||
1194             ((rdev->family) == CHIP_RV630) ||
1195             ((rdev->family) == CHIP_RV610) ||
1196             ((rdev->family) == CHIP_RV620) ||
1197             ((rdev->family) == CHIP_RS780) ||
1198             ((rdev->family) == CHIP_RS880)) {
1199                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1200         } else {
1201                 WREG32(DB_DEBUG, 0);
1202         }
1203         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1204                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1205
1206         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1207         WREG32(VGT_NUM_INSTANCES, 0);
1208
1209         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1210         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1211
1212         tmp = RREG32(SQ_MS_FIFO_SIZES);
1213         if (((rdev->family) == CHIP_RV610) ||
1214             ((rdev->family) == CHIP_RV620) ||
1215             ((rdev->family) == CHIP_RS780) ||
1216             ((rdev->family) == CHIP_RS880)) {
1217                 tmp = (CACHE_FIFO_SIZE(0xa) |
1218                        FETCH_FIFO_HIWATER(0xa) |
1219                        DONE_FIFO_HIWATER(0xe0) |
1220                        ALU_UPDATE_FIFO_HIWATER(0x8));
1221         } else if (((rdev->family) == CHIP_R600) ||
1222                    ((rdev->family) == CHIP_RV630)) {
1223                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1224                 tmp |= DONE_FIFO_HIWATER(0x4);
1225         }
1226         WREG32(SQ_MS_FIFO_SIZES, tmp);
1227
1228         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1229          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1230          */
1231         sq_config = RREG32(SQ_CONFIG);
1232         sq_config &= ~(PS_PRIO(3) |
1233                        VS_PRIO(3) |
1234                        GS_PRIO(3) |
1235                        ES_PRIO(3));
1236         sq_config |= (DX9_CONSTS |
1237                       VC_ENABLE |
1238                       PS_PRIO(0) |
1239                       VS_PRIO(1) |
1240                       GS_PRIO(2) |
1241                       ES_PRIO(3));
1242
1243         if ((rdev->family) == CHIP_R600) {
1244                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1245                                           NUM_VS_GPRS(124) |
1246                                           NUM_CLAUSE_TEMP_GPRS(4));
1247                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1248                                           NUM_ES_GPRS(0));
1249                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1250                                            NUM_VS_THREADS(48) |
1251                                            NUM_GS_THREADS(4) |
1252                                            NUM_ES_THREADS(4));
1253                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1254                                             NUM_VS_STACK_ENTRIES(128));
1255                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1256                                             NUM_ES_STACK_ENTRIES(0));
1257         } else if (((rdev->family) == CHIP_RV610) ||
1258                    ((rdev->family) == CHIP_RV620) ||
1259                    ((rdev->family) == CHIP_RS780) ||
1260                    ((rdev->family) == CHIP_RS880)) {
1261                 /* no vertex cache */
1262                 sq_config &= ~VC_ENABLE;
1263
1264                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1265                                           NUM_VS_GPRS(44) |
1266                                           NUM_CLAUSE_TEMP_GPRS(2));
1267                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1268                                           NUM_ES_GPRS(17));
1269                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1270                                            NUM_VS_THREADS(78) |
1271                                            NUM_GS_THREADS(4) |
1272                                            NUM_ES_THREADS(31));
1273                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1274                                             NUM_VS_STACK_ENTRIES(40));
1275                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1276                                             NUM_ES_STACK_ENTRIES(16));
1277         } else if (((rdev->family) == CHIP_RV630) ||
1278                    ((rdev->family) == CHIP_RV635)) {
1279                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1280                                           NUM_VS_GPRS(44) |
1281                                           NUM_CLAUSE_TEMP_GPRS(2));
1282                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1283                                           NUM_ES_GPRS(18));
1284                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1285                                            NUM_VS_THREADS(78) |
1286                                            NUM_GS_THREADS(4) |
1287                                            NUM_ES_THREADS(31));
1288                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1289                                             NUM_VS_STACK_ENTRIES(40));
1290                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1291                                             NUM_ES_STACK_ENTRIES(16));
1292         } else if ((rdev->family) == CHIP_RV670) {
1293                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1294                                           NUM_VS_GPRS(44) |
1295                                           NUM_CLAUSE_TEMP_GPRS(2));
1296                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1297                                           NUM_ES_GPRS(17));
1298                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1299                                            NUM_VS_THREADS(78) |
1300                                            NUM_GS_THREADS(4) |
1301                                            NUM_ES_THREADS(31));
1302                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1303                                             NUM_VS_STACK_ENTRIES(64));
1304                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1305                                             NUM_ES_STACK_ENTRIES(64));
1306         }
1307
1308         WREG32(SQ_CONFIG, sq_config);
1309         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1310         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1311         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1312         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1313         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1314
1315         if (((rdev->family) == CHIP_RV610) ||
1316             ((rdev->family) == CHIP_RV620) ||
1317             ((rdev->family) == CHIP_RS780) ||
1318             ((rdev->family) == CHIP_RS880)) {
1319                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1320         } else {
1321                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1322         }
1323
1324         /* More default values. 2D/3D driver should adjust as needed */
1325         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1326                                          S1_X(0x4) | S1_Y(0xc)));
1327         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1328                                          S1_X(0x2) | S1_Y(0x2) |
1329                                          S2_X(0xa) | S2_Y(0x6) |
1330                                          S3_X(0x6) | S3_Y(0xa)));
1331         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1332                                              S1_X(0x4) | S1_Y(0xc) |
1333                                              S2_X(0x1) | S2_Y(0x6) |
1334                                              S3_X(0xa) | S3_Y(0xe)));
1335         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1336                                              S5_X(0x0) | S5_Y(0x0) |
1337                                              S6_X(0xb) | S6_Y(0x4) |
1338                                              S7_X(0x7) | S7_Y(0x8)));
1339
1340         WREG32(VGT_STRMOUT_EN, 0);
1341         tmp = rdev->config.r600.max_pipes * 16;
1342         switch (rdev->family) {
1343         case CHIP_RV610:
1344         case CHIP_RV620:
1345         case CHIP_RS780:
1346         case CHIP_RS880:
1347                 tmp += 32;
1348                 break;
1349         case CHIP_RV670:
1350                 tmp += 128;
1351                 break;
1352         default:
1353                 break;
1354         }
1355         if (tmp > 256) {
1356                 tmp = 256;
1357         }
1358         WREG32(VGT_ES_PER_GS, 128);
1359         WREG32(VGT_GS_PER_ES, tmp);
1360         WREG32(VGT_GS_PER_VS, 2);
1361         WREG32(VGT_GS_VERTEX_REUSE, 16);
1362
1363         /* more default values. 2D/3D driver should adjust as needed */
1364         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1365         WREG32(VGT_STRMOUT_EN, 0);
1366         WREG32(SX_MISC, 0);
1367         WREG32(PA_SC_MODE_CNTL, 0);
1368         WREG32(PA_SC_AA_CONFIG, 0);
1369         WREG32(PA_SC_LINE_STIPPLE, 0);
1370         WREG32(SPI_INPUT_Z, 0);
1371         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1372         WREG32(CB_COLOR7_FRAG, 0);
1373
1374         /* Clear render buffer base addresses */
1375         WREG32(CB_COLOR0_BASE, 0);
1376         WREG32(CB_COLOR1_BASE, 0);
1377         WREG32(CB_COLOR2_BASE, 0);
1378         WREG32(CB_COLOR3_BASE, 0);
1379         WREG32(CB_COLOR4_BASE, 0);
1380         WREG32(CB_COLOR5_BASE, 0);
1381         WREG32(CB_COLOR6_BASE, 0);
1382         WREG32(CB_COLOR7_BASE, 0);
1383         WREG32(CB_COLOR7_FRAG, 0);
1384
1385         switch (rdev->family) {
1386         case CHIP_RV610:
1387         case CHIP_RV620:
1388         case CHIP_RS780:
1389         case CHIP_RS880:
1390                 tmp = TC_L2_SIZE(8);
1391                 break;
1392         case CHIP_RV630:
1393         case CHIP_RV635:
1394                 tmp = TC_L2_SIZE(4);
1395                 break;
1396         case CHIP_R600:
1397                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
1398                 break;
1399         default:
1400                 tmp = TC_L2_SIZE(0);
1401                 break;
1402         }
1403         WREG32(TC_CNTL, tmp);
1404
1405         tmp = RREG32(HDP_HOST_PATH_CNTL);
1406         WREG32(HDP_HOST_PATH_CNTL, tmp);
1407
1408         tmp = RREG32(ARB_POP);
1409         tmp |= ENABLE_TC128;
1410         WREG32(ARB_POP, tmp);
1411
1412         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1413         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1414                                NUM_CLIP_SEQ(3)));
1415         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
1416 }
1417
1418
1419 /*
1420  * Indirect registers accessor
1421  */
1422 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1423 {
1424         u32 r;
1425
1426         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1427         (void)RREG32(PCIE_PORT_INDEX);
1428         r = RREG32(PCIE_PORT_DATA);
1429         return r;
1430 }
1431
1432 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1433 {
1434         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1435         (void)RREG32(PCIE_PORT_INDEX);
1436         WREG32(PCIE_PORT_DATA, (v));
1437         (void)RREG32(PCIE_PORT_DATA);
1438 }
1439
1440 /*
1441  * CP & Ring
1442  */
1443 void r600_cp_stop(struct radeon_device *rdev)
1444 {
1445         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1446 }
1447
1448 int r600_init_microcode(struct radeon_device *rdev)
1449 {
1450         struct platform_device *pdev;
1451         const char *chip_name;
1452         const char *rlc_chip_name;
1453         size_t pfp_req_size, me_req_size, rlc_req_size;
1454         char fw_name[30];
1455         int err;
1456
1457         DRM_DEBUG("\n");
1458
1459         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1460         err = IS_ERR(pdev);
1461         if (err) {
1462                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1463                 return -EINVAL;
1464         }
1465
1466         switch (rdev->family) {
1467         case CHIP_R600:
1468                 chip_name = "R600";
1469                 rlc_chip_name = "R600";
1470                 break;
1471         case CHIP_RV610:
1472                 chip_name = "RV610";
1473                 rlc_chip_name = "R600";
1474                 break;
1475         case CHIP_RV630:
1476                 chip_name = "RV630";
1477                 rlc_chip_name = "R600";
1478                 break;
1479         case CHIP_RV620:
1480                 chip_name = "RV620";
1481                 rlc_chip_name = "R600";
1482                 break;
1483         case CHIP_RV635:
1484                 chip_name = "RV635";
1485                 rlc_chip_name = "R600";
1486                 break;
1487         case CHIP_RV670:
1488                 chip_name = "RV670";
1489                 rlc_chip_name = "R600";
1490                 break;
1491         case CHIP_RS780:
1492         case CHIP_RS880:
1493                 chip_name = "RS780";
1494                 rlc_chip_name = "R600";
1495                 break;
1496         case CHIP_RV770:
1497                 chip_name = "RV770";
1498                 rlc_chip_name = "R700";
1499                 break;
1500         case CHIP_RV730:
1501         case CHIP_RV740:
1502                 chip_name = "RV730";
1503                 rlc_chip_name = "R700";
1504                 break;
1505         case CHIP_RV710:
1506                 chip_name = "RV710";
1507                 rlc_chip_name = "R700";
1508                 break;
1509         case CHIP_CEDAR:
1510                 chip_name = "CEDAR";
1511                 rlc_chip_name = "CEDAR";
1512                 break;
1513         case CHIP_REDWOOD:
1514                 chip_name = "REDWOOD";
1515                 rlc_chip_name = "REDWOOD";
1516                 break;
1517         case CHIP_JUNIPER:
1518                 chip_name = "JUNIPER";
1519                 rlc_chip_name = "JUNIPER";
1520                 break;
1521         case CHIP_CYPRESS:
1522         case CHIP_HEMLOCK:
1523                 chip_name = "CYPRESS";
1524                 rlc_chip_name = "CYPRESS";
1525                 break;
1526         default: BUG();
1527         }
1528
1529         if (rdev->family >= CHIP_CEDAR) {
1530                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
1531                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
1532                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
1533         } else if (rdev->family >= CHIP_RV770) {
1534                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
1535                 me_req_size = R700_PM4_UCODE_SIZE * 4;
1536                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
1537         } else {
1538                 pfp_req_size = PFP_UCODE_SIZE * 4;
1539                 me_req_size = PM4_UCODE_SIZE * 12;
1540                 rlc_req_size = RLC_UCODE_SIZE * 4;
1541         }
1542
1543         DRM_INFO("Loading %s Microcode\n", chip_name);
1544
1545         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1546         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1547         if (err)
1548                 goto out;
1549         if (rdev->pfp_fw->size != pfp_req_size) {
1550                 printk(KERN_ERR
1551                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1552                        rdev->pfp_fw->size, fw_name);
1553                 err = -EINVAL;
1554                 goto out;
1555         }
1556
1557         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1558         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1559         if (err)
1560                 goto out;
1561         if (rdev->me_fw->size != me_req_size) {
1562                 printk(KERN_ERR
1563                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1564                        rdev->me_fw->size, fw_name);
1565                 err = -EINVAL;
1566         }
1567
1568         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1569         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1570         if (err)
1571                 goto out;
1572         if (rdev->rlc_fw->size != rlc_req_size) {
1573                 printk(KERN_ERR
1574                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
1575                        rdev->rlc_fw->size, fw_name);
1576                 err = -EINVAL;
1577         }
1578
1579 out:
1580         platform_device_unregister(pdev);
1581
1582         if (err) {
1583                 if (err != -EINVAL)
1584                         printk(KERN_ERR
1585                                "r600_cp: Failed to load firmware \"%s\"\n",
1586                                fw_name);
1587                 release_firmware(rdev->pfp_fw);
1588                 rdev->pfp_fw = NULL;
1589                 release_firmware(rdev->me_fw);
1590                 rdev->me_fw = NULL;
1591                 release_firmware(rdev->rlc_fw);
1592                 rdev->rlc_fw = NULL;
1593         }
1594         return err;
1595 }
1596
1597 static int r600_cp_load_microcode(struct radeon_device *rdev)
1598 {
1599         const __be32 *fw_data;
1600         int i;
1601
1602         if (!rdev->me_fw || !rdev->pfp_fw)
1603                 return -EINVAL;
1604
1605         r600_cp_stop(rdev);
1606
1607         WREG32(CP_RB_CNTL, RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1608
1609         /* Reset cp */
1610         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1611         RREG32(GRBM_SOFT_RESET);
1612         mdelay(15);
1613         WREG32(GRBM_SOFT_RESET, 0);
1614
1615         WREG32(CP_ME_RAM_WADDR, 0);
1616
1617         fw_data = (const __be32 *)rdev->me_fw->data;
1618         WREG32(CP_ME_RAM_WADDR, 0);
1619         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
1620                 WREG32(CP_ME_RAM_DATA,
1621                        be32_to_cpup(fw_data++));
1622
1623         fw_data = (const __be32 *)rdev->pfp_fw->data;
1624         WREG32(CP_PFP_UCODE_ADDR, 0);
1625         for (i = 0; i < PFP_UCODE_SIZE; i++)
1626                 WREG32(CP_PFP_UCODE_DATA,
1627                        be32_to_cpup(fw_data++));
1628
1629         WREG32(CP_PFP_UCODE_ADDR, 0);
1630         WREG32(CP_ME_RAM_WADDR, 0);
1631         WREG32(CP_ME_RAM_RADDR, 0);
1632         return 0;
1633 }
1634
1635 int r600_cp_start(struct radeon_device *rdev)
1636 {
1637         int r;
1638         uint32_t cp_me;
1639
1640         r = radeon_ring_lock(rdev, 7);
1641         if (r) {
1642                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1643                 return r;
1644         }
1645         radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1646         radeon_ring_write(rdev, 0x1);
1647         if (rdev->family >= CHIP_CEDAR) {
1648                 radeon_ring_write(rdev, 0x0);
1649                 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1650         } else if (rdev->family >= CHIP_RV770) {
1651                 radeon_ring_write(rdev, 0x0);
1652                 radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
1653         } else {
1654                 radeon_ring_write(rdev, 0x3);
1655                 radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
1656         }
1657         radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1658         radeon_ring_write(rdev, 0);
1659         radeon_ring_write(rdev, 0);
1660         radeon_ring_unlock_commit(rdev);
1661
1662         cp_me = 0xff;
1663         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
1664         return 0;
1665 }
1666
1667 int r600_cp_resume(struct radeon_device *rdev)
1668 {
1669         u32 tmp;
1670         u32 rb_bufsz;
1671         int r;
1672
1673         /* Reset cp */
1674         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1675         RREG32(GRBM_SOFT_RESET);
1676         mdelay(15);
1677         WREG32(GRBM_SOFT_RESET, 0);
1678
1679         /* Set ring buffer size */
1680         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1681         tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1682 #ifdef __BIG_ENDIAN
1683         tmp |= BUF_SWAP_32BIT;
1684 #endif
1685         WREG32(CP_RB_CNTL, tmp);
1686         WREG32(CP_SEM_WAIT_TIMER, 0x4);
1687
1688         /* Set the write pointer delay */
1689         WREG32(CP_RB_WPTR_DELAY, 0);
1690
1691         /* Initialize the ring buffer's read and write pointers */
1692         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1693         WREG32(CP_RB_RPTR_WR, 0);
1694         WREG32(CP_RB_WPTR, 0);
1695         WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
1696         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
1697         mdelay(1);
1698         WREG32(CP_RB_CNTL, tmp);
1699
1700         WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1701         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1702
1703         rdev->cp.rptr = RREG32(CP_RB_RPTR);
1704         rdev->cp.wptr = RREG32(CP_RB_WPTR);
1705
1706         r600_cp_start(rdev);
1707         rdev->cp.ready = true;
1708         r = radeon_ring_test(rdev);
1709         if (r) {
1710                 rdev->cp.ready = false;
1711                 return r;
1712         }
1713         return 0;
1714 }
1715
1716 void r600_cp_commit(struct radeon_device *rdev)
1717 {
1718         WREG32(CP_RB_WPTR, rdev->cp.wptr);
1719         (void)RREG32(CP_RB_WPTR);
1720 }
1721
1722 void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
1723 {
1724         u32 rb_bufsz;
1725
1726         /* Align ring size */
1727         rb_bufsz = drm_order(ring_size / 8);
1728         ring_size = (1 << (rb_bufsz + 1)) * 4;
1729         rdev->cp.ring_size = ring_size;
1730         rdev->cp.align_mask = 16 - 1;
1731 }
1732
1733 void r600_cp_fini(struct radeon_device *rdev)
1734 {
1735         r600_cp_stop(rdev);
1736         radeon_ring_fini(rdev);
1737 }
1738
1739
1740 /*
1741  * GPU scratch registers helpers function.
1742  */
1743 void r600_scratch_init(struct radeon_device *rdev)
1744 {
1745         int i;
1746
1747         rdev->scratch.num_reg = 7;
1748         for (i = 0; i < rdev->scratch.num_reg; i++) {
1749                 rdev->scratch.free[i] = true;
1750                 rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4);
1751         }
1752 }
1753
1754 int r600_ring_test(struct radeon_device *rdev)
1755 {
1756         uint32_t scratch;
1757         uint32_t tmp = 0;
1758         unsigned i;
1759         int r;
1760
1761         r = radeon_scratch_get(rdev, &scratch);
1762         if (r) {
1763                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
1764                 return r;
1765         }
1766         WREG32(scratch, 0xCAFEDEAD);
1767         r = radeon_ring_lock(rdev, 3);
1768         if (r) {
1769                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1770                 radeon_scratch_free(rdev, scratch);
1771                 return r;
1772         }
1773         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1774         radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1775         radeon_ring_write(rdev, 0xDEADBEEF);
1776         radeon_ring_unlock_commit(rdev);
1777         for (i = 0; i < rdev->usec_timeout; i++) {
1778                 tmp = RREG32(scratch);
1779                 if (tmp == 0xDEADBEEF)
1780                         break;
1781                 DRM_UDELAY(1);
1782         }
1783         if (i < rdev->usec_timeout) {
1784                 DRM_INFO("ring test succeeded in %d usecs\n", i);
1785         } else {
1786                 DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
1787                           scratch, tmp);
1788                 r = -EINVAL;
1789         }
1790         radeon_scratch_free(rdev, scratch);
1791         return r;
1792 }
1793
1794 void r600_wb_disable(struct radeon_device *rdev)
1795 {
1796         int r;
1797
1798         WREG32(SCRATCH_UMSK, 0);
1799         if (rdev->wb.wb_obj) {
1800                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
1801                 if (unlikely(r != 0))
1802                         return;
1803                 radeon_bo_kunmap(rdev->wb.wb_obj);
1804                 radeon_bo_unpin(rdev->wb.wb_obj);
1805                 radeon_bo_unreserve(rdev->wb.wb_obj);
1806         }
1807 }
1808
1809 void r600_wb_fini(struct radeon_device *rdev)
1810 {
1811         r600_wb_disable(rdev);
1812         if (rdev->wb.wb_obj) {
1813                 radeon_bo_unref(&rdev->wb.wb_obj);
1814                 rdev->wb.wb = NULL;
1815                 rdev->wb.wb_obj = NULL;
1816         }
1817 }
1818
1819 int r600_wb_enable(struct radeon_device *rdev)
1820 {
1821         int r;
1822
1823         if (rdev->wb.wb_obj == NULL) {
1824                 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
1825                                 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
1826                 if (r) {
1827                         dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
1828                         return r;
1829                 }
1830                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
1831                 if (unlikely(r != 0)) {
1832                         r600_wb_fini(rdev);
1833                         return r;
1834                 }
1835                 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
1836                                 &rdev->wb.gpu_addr);
1837                 if (r) {
1838                         radeon_bo_unreserve(rdev->wb.wb_obj);
1839                         dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
1840                         r600_wb_fini(rdev);
1841                         return r;
1842                 }
1843                 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
1844                 radeon_bo_unreserve(rdev->wb.wb_obj);
1845                 if (r) {
1846                         dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
1847                         r600_wb_fini(rdev);
1848                         return r;
1849                 }
1850         }
1851         WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
1852         WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
1853         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
1854         WREG32(SCRATCH_UMSK, 0xff);
1855         return 0;
1856 }
1857
1858 void r600_fence_ring_emit(struct radeon_device *rdev,
1859                           struct radeon_fence *fence)
1860 {
1861         /* Also consider EVENT_WRITE_EOP.  it handles the interrupts + timestamps + events */
1862
1863         radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
1864         radeon_ring_write(rdev, CACHE_FLUSH_AND_INV_EVENT);
1865         /* wait for 3D idle clean */
1866         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1867         radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
1868         radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
1869         /* Emit fence sequence & fire IRQ */
1870         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1871         radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1872         radeon_ring_write(rdev, fence->seq);
1873         /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
1874         radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
1875         radeon_ring_write(rdev, RB_INT_STAT);
1876 }
1877
1878 int r600_copy_blit(struct radeon_device *rdev,
1879                    uint64_t src_offset, uint64_t dst_offset,
1880                    unsigned num_pages, struct radeon_fence *fence)
1881 {
1882         int r;
1883
1884         mutex_lock(&rdev->r600_blit.mutex);
1885         rdev->r600_blit.vb_ib = NULL;
1886         r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
1887         if (r) {
1888                 if (rdev->r600_blit.vb_ib)
1889                         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
1890                 mutex_unlock(&rdev->r600_blit.mutex);
1891                 return r;
1892         }
1893         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
1894         r600_blit_done_copy(rdev, fence);
1895         mutex_unlock(&rdev->r600_blit.mutex);
1896         return 0;
1897 }
1898
1899 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
1900                          uint32_t tiling_flags, uint32_t pitch,
1901                          uint32_t offset, uint32_t obj_size)
1902 {
1903         /* FIXME: implement */
1904         return 0;
1905 }
1906
1907 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
1908 {
1909         /* FIXME: implement */
1910 }
1911
1912
1913 bool r600_card_posted(struct radeon_device *rdev)
1914 {
1915         uint32_t reg;
1916
1917         /* first check CRTCs */
1918         reg = RREG32(D1CRTC_CONTROL) |
1919                 RREG32(D2CRTC_CONTROL);
1920         if (reg & CRTC_EN)
1921                 return true;
1922
1923         /* then check MEM_SIZE, in case the crtcs are off */
1924         if (RREG32(CONFIG_MEMSIZE))
1925                 return true;
1926
1927         return false;
1928 }
1929
1930 int r600_startup(struct radeon_device *rdev)
1931 {
1932         int r;
1933
1934         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1935                 r = r600_init_microcode(rdev);
1936                 if (r) {
1937                         DRM_ERROR("Failed to load firmware!\n");
1938                         return r;
1939                 }
1940         }
1941
1942         r600_mc_program(rdev);
1943         if (rdev->flags & RADEON_IS_AGP) {
1944                 r600_agp_enable(rdev);
1945         } else {
1946                 r = r600_pcie_gart_enable(rdev);
1947                 if (r)
1948                         return r;
1949         }
1950         r600_gpu_init(rdev);
1951         r = r600_blit_init(rdev);
1952         if (r) {
1953                 r600_blit_fini(rdev);
1954                 rdev->asic->copy = NULL;
1955                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
1956         }
1957         /* pin copy shader into vram */
1958         if (rdev->r600_blit.shader_obj) {
1959                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
1960                 if (unlikely(r != 0))
1961                         return r;
1962                 r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
1963                                 &rdev->r600_blit.shader_gpu_addr);
1964                 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
1965                 if (r) {
1966                         dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
1967                         return r;
1968                 }
1969         }
1970         /* Enable IRQ */
1971         r = r600_irq_init(rdev);
1972         if (r) {
1973                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1974                 radeon_irq_kms_fini(rdev);
1975                 return r;
1976         }
1977         r600_irq_set(rdev);
1978
1979         r = radeon_ring_init(rdev, rdev->cp.ring_size);
1980         if (r)
1981                 return r;
1982         r = r600_cp_load_microcode(rdev);
1983         if (r)
1984                 return r;
1985         r = r600_cp_resume(rdev);
1986         if (r)
1987                 return r;
1988         /* write back buffer are not vital so don't worry about failure */
1989         r600_wb_enable(rdev);
1990         return 0;
1991 }
1992
1993 void r600_vga_set_state(struct radeon_device *rdev, bool state)
1994 {
1995         uint32_t temp;
1996
1997         temp = RREG32(CONFIG_CNTL);
1998         if (state == false) {
1999                 temp &= ~(1<<0);
2000                 temp |= (1<<1);
2001         } else {
2002                 temp &= ~(1<<1);
2003         }
2004         WREG32(CONFIG_CNTL, temp);
2005 }
2006
2007 int r600_resume(struct radeon_device *rdev)
2008 {
2009         int r;
2010
2011         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
2012          * posting will perform necessary task to bring back GPU into good
2013          * shape.
2014          */
2015         /* post card */
2016         atom_asic_init(rdev->mode_info.atom_context);
2017         /* Initialize clocks */
2018         r = radeon_clocks_init(rdev);
2019         if (r) {
2020                 return r;
2021         }
2022
2023         r = r600_startup(rdev);
2024         if (r) {
2025                 DRM_ERROR("r600 startup failed on resume\n");
2026                 return r;
2027         }
2028
2029         r = r600_ib_test(rdev);
2030         if (r) {
2031                 DRM_ERROR("radeon: failled testing IB (%d).\n", r);
2032                 return r;
2033         }
2034
2035         r = r600_audio_init(rdev);
2036         if (r) {
2037                 DRM_ERROR("radeon: audio resume failed\n");
2038                 return r;
2039         }
2040
2041         return r;
2042 }
2043
2044 int r600_suspend(struct radeon_device *rdev)
2045 {
2046         int r;
2047
2048         r600_audio_fini(rdev);
2049         /* FIXME: we should wait for ring to be empty */
2050         r600_cp_stop(rdev);
2051         rdev->cp.ready = false;
2052         r600_irq_suspend(rdev);
2053         r600_wb_disable(rdev);
2054         r600_pcie_gart_disable(rdev);
2055         /* unpin shaders bo */
2056         if (rdev->r600_blit.shader_obj) {
2057                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
2058                 if (!r) {
2059                         radeon_bo_unpin(rdev->r600_blit.shader_obj);
2060                         radeon_bo_unreserve(rdev->r600_blit.shader_obj);
2061                 }
2062         }
2063         return 0;
2064 }
2065
2066 /* Plan is to move initialization in that function and use
2067  * helper function so that radeon_device_init pretty much
2068  * do nothing more than calling asic specific function. This
2069  * should also allow to remove a bunch of callback function
2070  * like vram_info.
2071  */
2072 int r600_init(struct radeon_device *rdev)
2073 {
2074         int r;
2075
2076         r = radeon_dummy_page_init(rdev);
2077         if (r)
2078                 return r;
2079         if (r600_debugfs_mc_info_init(rdev)) {
2080                 DRM_ERROR("Failed to register debugfs file for mc !\n");
2081         }
2082         /* This don't do much */
2083         r = radeon_gem_init(rdev);
2084         if (r)
2085                 return r;
2086         /* Read BIOS */
2087         if (!radeon_get_bios(rdev)) {
2088                 if (ASIC_IS_AVIVO(rdev))
2089                         return -EINVAL;
2090         }
2091         /* Must be an ATOMBIOS */
2092         if (!rdev->is_atom_bios) {
2093                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
2094                 return -EINVAL;
2095         }
2096         r = radeon_atombios_init(rdev);
2097         if (r)
2098                 return r;
2099         /* Post card if necessary */
2100         if (!r600_card_posted(rdev)) {
2101                 if (!rdev->bios) {
2102                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2103                         return -EINVAL;
2104                 }
2105                 DRM_INFO("GPU not posted. posting now...\n");
2106                 atom_asic_init(rdev->mode_info.atom_context);
2107         }
2108         /* Initialize scratch registers */
2109         r600_scratch_init(rdev);
2110         /* Initialize surface registers */
2111         radeon_surface_init(rdev);
2112         /* Initialize clocks */
2113         radeon_get_clock_info(rdev->ddev);
2114         r = radeon_clocks_init(rdev);
2115         if (r)
2116                 return r;
2117         /* Initialize power management */
2118         radeon_pm_init(rdev);
2119         /* Fence driver */
2120         r = radeon_fence_driver_init(rdev);
2121         if (r)
2122                 return r;
2123         if (rdev->flags & RADEON_IS_AGP) {
2124                 r = radeon_agp_init(rdev);
2125                 if (r)
2126                         radeon_agp_disable(rdev);
2127         }
2128         r = r600_mc_init(rdev);
2129         if (r)
2130                 return r;
2131         /* Memory manager */
2132         r = radeon_bo_init(rdev);
2133         if (r)
2134                 return r;
2135
2136         r = radeon_irq_kms_init(rdev);
2137         if (r)
2138                 return r;
2139
2140         rdev->cp.ring_obj = NULL;
2141         r600_ring_init(rdev, 1024 * 1024);
2142
2143         rdev->ih.ring_obj = NULL;
2144         r600_ih_ring_init(rdev, 64 * 1024);
2145
2146         r = r600_pcie_gart_init(rdev);
2147         if (r)
2148                 return r;
2149
2150         rdev->accel_working = true;
2151         r = r600_startup(rdev);
2152         if (r) {
2153                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2154                 r600_cp_fini(rdev);
2155                 r600_wb_fini(rdev);
2156                 r600_irq_fini(rdev);
2157                 radeon_irq_kms_fini(rdev);
2158                 r600_pcie_gart_fini(rdev);
2159                 rdev->accel_working = false;
2160         }
2161         if (rdev->accel_working) {
2162                 r = radeon_ib_pool_init(rdev);
2163                 if (r) {
2164                         dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2165                         rdev->accel_working = false;
2166                 } else {
2167                         r = r600_ib_test(rdev);
2168                         if (r) {
2169                                 dev_err(rdev->dev, "IB test failed (%d).\n", r);
2170                                 rdev->accel_working = false;
2171                         }
2172                 }
2173         }
2174
2175         r = r600_audio_init(rdev);
2176         if (r)
2177                 return r; /* TODO error handling */
2178         return 0;
2179 }
2180
2181 void r600_fini(struct radeon_device *rdev)
2182 {
2183         radeon_pm_fini(rdev);
2184         r600_audio_fini(rdev);
2185         r600_blit_fini(rdev);
2186         r600_cp_fini(rdev);
2187         r600_wb_fini(rdev);
2188         r600_irq_fini(rdev);
2189         radeon_irq_kms_fini(rdev);
2190         r600_pcie_gart_fini(rdev);
2191         radeon_agp_fini(rdev);
2192         radeon_gem_fini(rdev);
2193         radeon_fence_driver_fini(rdev);
2194         radeon_clocks_fini(rdev);
2195         radeon_bo_fini(rdev);
2196         radeon_atombios_fini(rdev);
2197         kfree(rdev->bios);
2198         rdev->bios = NULL;
2199         radeon_dummy_page_fini(rdev);
2200 }
2201
2202
2203 /*
2204  * CS stuff
2205  */
2206 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2207 {
2208         /* FIXME: implement */
2209         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2210         radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
2211         radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
2212         radeon_ring_write(rdev, ib->length_dw);
2213 }
2214
2215 int r600_ib_test(struct radeon_device *rdev)
2216 {
2217         struct radeon_ib *ib;
2218         uint32_t scratch;
2219         uint32_t tmp = 0;
2220         unsigned i;
2221         int r;
2222
2223         r = radeon_scratch_get(rdev, &scratch);
2224         if (r) {
2225                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
2226                 return r;
2227         }
2228         WREG32(scratch, 0xCAFEDEAD);
2229         r = radeon_ib_get(rdev, &ib);
2230         if (r) {
2231                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2232                 return r;
2233         }
2234         ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
2235         ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2236         ib->ptr[2] = 0xDEADBEEF;
2237         ib->ptr[3] = PACKET2(0);
2238         ib->ptr[4] = PACKET2(0);
2239         ib->ptr[5] = PACKET2(0);
2240         ib->ptr[6] = PACKET2(0);
2241         ib->ptr[7] = PACKET2(0);
2242         ib->ptr[8] = PACKET2(0);
2243         ib->ptr[9] = PACKET2(0);
2244         ib->ptr[10] = PACKET2(0);
2245         ib->ptr[11] = PACKET2(0);
2246         ib->ptr[12] = PACKET2(0);
2247         ib->ptr[13] = PACKET2(0);
2248         ib->ptr[14] = PACKET2(0);
2249         ib->ptr[15] = PACKET2(0);
2250         ib->length_dw = 16;
2251         r = radeon_ib_schedule(rdev, ib);
2252         if (r) {
2253                 radeon_scratch_free(rdev, scratch);
2254                 radeon_ib_free(rdev, &ib);
2255                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2256                 return r;
2257         }
2258         r = radeon_fence_wait(ib->fence, false);
2259         if (r) {
2260                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2261                 return r;
2262         }
2263         for (i = 0; i < rdev->usec_timeout; i++) {
2264                 tmp = RREG32(scratch);
2265                 if (tmp == 0xDEADBEEF)
2266                         break;
2267                 DRM_UDELAY(1);
2268         }
2269         if (i < rdev->usec_timeout) {
2270                 DRM_INFO("ib test succeeded in %u usecs\n", i);
2271         } else {
2272                 DRM_ERROR("radeon: ib test failed (sracth(0x%04X)=0x%08X)\n",
2273                           scratch, tmp);
2274                 r = -EINVAL;
2275         }
2276         radeon_scratch_free(rdev, scratch);
2277         radeon_ib_free(rdev, &ib);
2278         return r;
2279 }
2280
2281 /*
2282  * Interrupts
2283  *
2284  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
2285  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
2286  * writing to the ring and the GPU consuming, the GPU writes to the ring
2287  * and host consumes.  As the host irq handler processes interrupts, it
2288  * increments the rptr.  When the rptr catches up with the wptr, all the
2289  * current interrupts have been processed.
2290  */
2291
2292 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
2293 {
2294         u32 rb_bufsz;
2295
2296         /* Align ring size */
2297         rb_bufsz = drm_order(ring_size / 4);
2298         ring_size = (1 << rb_bufsz) * 4;
2299         rdev->ih.ring_size = ring_size;
2300         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
2301         rdev->ih.rptr = 0;
2302 }
2303
2304 static int r600_ih_ring_alloc(struct radeon_device *rdev)
2305 {
2306         int r;
2307
2308         /* Allocate ring buffer */
2309         if (rdev->ih.ring_obj == NULL) {
2310                 r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
2311                                      true,
2312                                      RADEON_GEM_DOMAIN_GTT,
2313                                      &rdev->ih.ring_obj);
2314                 if (r) {
2315                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
2316                         return r;
2317                 }
2318                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2319                 if (unlikely(r != 0))
2320                         return r;
2321                 r = radeon_bo_pin(rdev->ih.ring_obj,
2322                                   RADEON_GEM_DOMAIN_GTT,
2323                                   &rdev->ih.gpu_addr);
2324                 if (r) {
2325                         radeon_bo_unreserve(rdev->ih.ring_obj);
2326                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
2327                         return r;
2328                 }
2329                 r = radeon_bo_kmap(rdev->ih.ring_obj,
2330                                    (void **)&rdev->ih.ring);
2331                 radeon_bo_unreserve(rdev->ih.ring_obj);
2332                 if (r) {
2333                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
2334                         return r;
2335                 }
2336         }
2337         return 0;
2338 }
2339
2340 static void r600_ih_ring_fini(struct radeon_device *rdev)
2341 {
2342         int r;
2343         if (rdev->ih.ring_obj) {
2344                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2345                 if (likely(r == 0)) {
2346                         radeon_bo_kunmap(rdev->ih.ring_obj);
2347                         radeon_bo_unpin(rdev->ih.ring_obj);
2348                         radeon_bo_unreserve(rdev->ih.ring_obj);
2349                 }
2350                 radeon_bo_unref(&rdev->ih.ring_obj);
2351                 rdev->ih.ring = NULL;
2352                 rdev->ih.ring_obj = NULL;
2353         }
2354 }
2355
2356 void r600_rlc_stop(struct radeon_device *rdev)
2357 {
2358
2359         if ((rdev->family >= CHIP_RV770) &&
2360             (rdev->family <= CHIP_RV740)) {
2361                 /* r7xx asics need to soft reset RLC before halting */
2362                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
2363                 RREG32(SRBM_SOFT_RESET);
2364                 udelay(15000);
2365                 WREG32(SRBM_SOFT_RESET, 0);
2366                 RREG32(SRBM_SOFT_RESET);
2367         }
2368
2369         WREG32(RLC_CNTL, 0);
2370 }
2371
2372 static void r600_rlc_start(struct radeon_device *rdev)
2373 {
2374         WREG32(RLC_CNTL, RLC_ENABLE);
2375 }
2376
2377 static int r600_rlc_init(struct radeon_device *rdev)
2378 {
2379         u32 i;
2380         const __be32 *fw_data;
2381
2382         if (!rdev->rlc_fw)
2383                 return -EINVAL;
2384
2385         r600_rlc_stop(rdev);
2386
2387         WREG32(RLC_HB_BASE, 0);
2388         WREG32(RLC_HB_CNTL, 0);
2389         WREG32(RLC_HB_RPTR, 0);
2390         WREG32(RLC_HB_WPTR, 0);
2391         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
2392         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2393         WREG32(RLC_MC_CNTL, 0);
2394         WREG32(RLC_UCODE_CNTL, 0);
2395
2396         fw_data = (const __be32 *)rdev->rlc_fw->data;
2397         if (rdev->family >= CHIP_CEDAR) {
2398                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
2399                         WREG32(RLC_UCODE_ADDR, i);
2400                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2401                 }
2402         } else if (rdev->family >= CHIP_RV770) {
2403                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
2404                         WREG32(RLC_UCODE_ADDR, i);
2405                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2406                 }
2407         } else {
2408                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
2409                         WREG32(RLC_UCODE_ADDR, i);
2410                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2411                 }
2412         }
2413         WREG32(RLC_UCODE_ADDR, 0);
2414
2415         r600_rlc_start(rdev);
2416
2417         return 0;
2418 }
2419
2420 static void r600_enable_interrupts(struct radeon_device *rdev)
2421 {
2422         u32 ih_cntl = RREG32(IH_CNTL);
2423         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2424
2425         ih_cntl |= ENABLE_INTR;
2426         ih_rb_cntl |= IH_RB_ENABLE;
2427         WREG32(IH_CNTL, ih_cntl);
2428         WREG32(IH_RB_CNTL, ih_rb_cntl);
2429         rdev->ih.enabled = true;
2430 }
2431
2432 void r600_disable_interrupts(struct radeon_device *rdev)
2433 {
2434         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2435         u32 ih_cntl = RREG32(IH_CNTL);
2436
2437         ih_rb_cntl &= ~IH_RB_ENABLE;
2438         ih_cntl &= ~ENABLE_INTR;
2439         WREG32(IH_RB_CNTL, ih_rb_cntl);
2440         WREG32(IH_CNTL, ih_cntl);
2441         /* set rptr, wptr to 0 */
2442         WREG32(IH_RB_RPTR, 0);
2443         WREG32(IH_RB_WPTR, 0);
2444         rdev->ih.enabled = false;
2445         rdev->ih.wptr = 0;
2446         rdev->ih.rptr = 0;
2447 }
2448
2449 static void r600_disable_interrupt_state(struct radeon_device *rdev)
2450 {
2451         u32 tmp;
2452
2453         WREG32(CP_INT_CNTL, 0);
2454         WREG32(GRBM_INT_CNTL, 0);
2455         WREG32(DxMODE_INT_MASK, 0);
2456         if (ASIC_IS_DCE3(rdev)) {
2457                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
2458                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
2459                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2460                 WREG32(DC_HPD1_INT_CONTROL, tmp);
2461                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2462                 WREG32(DC_HPD2_INT_CONTROL, tmp);
2463                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2464                 WREG32(DC_HPD3_INT_CONTROL, tmp);
2465                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2466                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2467                 if (ASIC_IS_DCE32(rdev)) {
2468                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2469                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2470                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2471                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2472                 }
2473         } else {
2474                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2475                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2476                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2477                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2478                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2479                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2480                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2481                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2482         }
2483 }
2484
2485 int r600_irq_init(struct radeon_device *rdev)
2486 {
2487         int ret = 0;
2488         int rb_bufsz;
2489         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
2490
2491         /* allocate ring */
2492         ret = r600_ih_ring_alloc(rdev);
2493         if (ret)
2494                 return ret;
2495
2496         /* disable irqs */
2497         r600_disable_interrupts(rdev);
2498
2499         /* init rlc */
2500         ret = r600_rlc_init(rdev);
2501         if (ret) {
2502                 r600_ih_ring_fini(rdev);
2503                 return ret;
2504         }
2505
2506         /* setup interrupt control */
2507         /* set dummy read address to ring address */
2508         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
2509         interrupt_cntl = RREG32(INTERRUPT_CNTL);
2510         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
2511          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
2512          */
2513         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
2514         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
2515         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
2516         WREG32(INTERRUPT_CNTL, interrupt_cntl);
2517
2518         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
2519         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
2520
2521         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
2522                       IH_WPTR_OVERFLOW_CLEAR |
2523                       (rb_bufsz << 1));
2524         /* WPTR writeback, not yet */
2525         /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/
2526         WREG32(IH_RB_WPTR_ADDR_LO, 0);
2527         WREG32(IH_RB_WPTR_ADDR_HI, 0);
2528
2529         WREG32(IH_RB_CNTL, ih_rb_cntl);
2530
2531         /* set rptr, wptr to 0 */
2532         WREG32(IH_RB_RPTR, 0);
2533         WREG32(IH_RB_WPTR, 0);
2534
2535         /* Default settings for IH_CNTL (disabled at first) */
2536         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
2537         /* RPTR_REARM only works if msi's are enabled */
2538         if (rdev->msi_enabled)
2539                 ih_cntl |= RPTR_REARM;
2540
2541 #ifdef __BIG_ENDIAN
2542         ih_cntl |= IH_MC_SWAP(IH_MC_SWAP_32BIT);
2543 #endif
2544         WREG32(IH_CNTL, ih_cntl);
2545
2546         /* force the active interrupt state to all disabled */
2547         if (rdev->family >= CHIP_CEDAR)
2548                 evergreen_disable_interrupt_state(rdev);
2549         else
2550                 r600_disable_interrupt_state(rdev);
2551
2552         /* enable irqs */
2553         r600_enable_interrupts(rdev);
2554
2555         return ret;
2556 }
2557
2558 void r600_irq_suspend(struct radeon_device *rdev)
2559 {
2560         r600_irq_disable(rdev);
2561         r600_rlc_stop(rdev);
2562 }
2563
2564 void r600_irq_fini(struct radeon_device *rdev)
2565 {
2566         r600_irq_suspend(rdev);
2567         r600_ih_ring_fini(rdev);
2568 }
2569
2570 int r600_irq_set(struct radeon_device *rdev)
2571 {
2572         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2573         u32 mode_int = 0;
2574         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2575         u32 grbm_int_cntl = 0;
2576         u32 hdmi1, hdmi2;
2577
2578         if (!rdev->irq.installed) {
2579                 WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
2580                 return -EINVAL;
2581         }
2582         /* don't enable anything if the ih is disabled */
2583         if (!rdev->ih.enabled) {
2584                 r600_disable_interrupts(rdev);
2585                 /* force the active interrupt state to all disabled */
2586                 r600_disable_interrupt_state(rdev);
2587                 return 0;
2588         }
2589
2590         hdmi1 = RREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2591         if (ASIC_IS_DCE3(rdev)) {
2592                 hdmi2 = RREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2593                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2594                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2595                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2596                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2597                 if (ASIC_IS_DCE32(rdev)) {
2598                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2599                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2600                 }
2601         } else {
2602                 hdmi2 = RREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2603                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2604                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2605                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2606         }
2607
2608         if (rdev->irq.sw_int) {
2609                 DRM_DEBUG("r600_irq_set: sw int\n");
2610                 cp_int_cntl |= RB_INT_ENABLE;
2611         }
2612         if (rdev->irq.crtc_vblank_int[0]) {
2613                 DRM_DEBUG("r600_irq_set: vblank 0\n");
2614                 mode_int |= D1MODE_VBLANK_INT_MASK;
2615         }
2616         if (rdev->irq.crtc_vblank_int[1]) {
2617                 DRM_DEBUG("r600_irq_set: vblank 1\n");
2618                 mode_int |= D2MODE_VBLANK_INT_MASK;
2619         }
2620         if (rdev->irq.hpd[0]) {
2621                 DRM_DEBUG("r600_irq_set: hpd 1\n");
2622                 hpd1 |= DC_HPDx_INT_EN;
2623         }
2624         if (rdev->irq.hpd[1]) {
2625                 DRM_DEBUG("r600_irq_set: hpd 2\n");
2626                 hpd2 |= DC_HPDx_INT_EN;
2627         }
2628         if (rdev->irq.hpd[2]) {
2629                 DRM_DEBUG("r600_irq_set: hpd 3\n");
2630                 hpd3 |= DC_HPDx_INT_EN;
2631         }
2632         if (rdev->irq.hpd[3]) {
2633                 DRM_DEBUG("r600_irq_set: hpd 4\n");
2634                 hpd4 |= DC_HPDx_INT_EN;
2635         }
2636         if (rdev->irq.hpd[4]) {
2637                 DRM_DEBUG("r600_irq_set: hpd 5\n");
2638                 hpd5 |= DC_HPDx_INT_EN;
2639         }
2640         if (rdev->irq.hpd[5]) {
2641                 DRM_DEBUG("r600_irq_set: hpd 6\n");
2642                 hpd6 |= DC_HPDx_INT_EN;
2643         }
2644         if (rdev->irq.hdmi[0]) {
2645                 DRM_DEBUG("r600_irq_set: hdmi 1\n");
2646                 hdmi1 |= R600_HDMI_INT_EN;
2647         }
2648         if (rdev->irq.hdmi[1]) {
2649                 DRM_DEBUG("r600_irq_set: hdmi 2\n");
2650                 hdmi2 |= R600_HDMI_INT_EN;
2651         }
2652         if (rdev->irq.gui_idle) {
2653                 DRM_DEBUG("gui idle\n");
2654                 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2655         }
2656
2657         WREG32(CP_INT_CNTL, cp_int_cntl);
2658         WREG32(DxMODE_INT_MASK, mode_int);
2659         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2660         WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
2661         if (ASIC_IS_DCE3(rdev)) {
2662                 WREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, hdmi2);
2663                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2664                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2665                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2666                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2667                 if (ASIC_IS_DCE32(rdev)) {
2668                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
2669                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
2670                 }
2671         } else {
2672                 WREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, hdmi2);
2673                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
2674                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
2675                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2676         }
2677
2678         return 0;
2679 }
2680
2681 static inline void r600_irq_ack(struct radeon_device *rdev,
2682                                 u32 *disp_int,
2683                                 u32 *disp_int_cont,
2684                                 u32 *disp_int_cont2)
2685 {
2686         u32 tmp;
2687
2688         if (ASIC_IS_DCE3(rdev)) {
2689                 *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
2690                 *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
2691                 *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2692         } else {
2693                 *disp_int = RREG32(DISP_INTERRUPT_STATUS);
2694                 *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2695                 *disp_int_cont2 = 0;
2696         }
2697
2698         if (*disp_int & LB_D1_VBLANK_INTERRUPT)
2699                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2700         if (*disp_int & LB_D1_VLINE_INTERRUPT)
2701                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2702         if (*disp_int & LB_D2_VBLANK_INTERRUPT)
2703                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2704         if (*disp_int & LB_D2_VLINE_INTERRUPT)
2705                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2706         if (*disp_int & DC_HPD1_INTERRUPT) {
2707                 if (ASIC_IS_DCE3(rdev)) {
2708                         tmp = RREG32(DC_HPD1_INT_CONTROL);
2709                         tmp |= DC_HPDx_INT_ACK;
2710                         WREG32(DC_HPD1_INT_CONTROL, tmp);
2711                 } else {
2712                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
2713                         tmp |= DC_HPDx_INT_ACK;
2714                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2715                 }
2716         }
2717         if (*disp_int & DC_HPD2_INTERRUPT) {
2718                 if (ASIC_IS_DCE3(rdev)) {
2719                         tmp = RREG32(DC_HPD2_INT_CONTROL);
2720                         tmp |= DC_HPDx_INT_ACK;
2721                         WREG32(DC_HPD2_INT_CONTROL, tmp);
2722                 } else {
2723                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
2724                         tmp |= DC_HPDx_INT_ACK;
2725                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2726                 }
2727         }
2728         if (*disp_int_cont & DC_HPD3_INTERRUPT) {
2729                 if (ASIC_IS_DCE3(rdev)) {
2730                         tmp = RREG32(DC_HPD3_INT_CONTROL);
2731                         tmp |= DC_HPDx_INT_ACK;
2732                         WREG32(DC_HPD3_INT_CONTROL, tmp);
2733                 } else {
2734                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
2735                         tmp |= DC_HPDx_INT_ACK;
2736                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2737                 }
2738         }
2739         if (*disp_int_cont & DC_HPD4_INTERRUPT) {
2740                 tmp = RREG32(DC_HPD4_INT_CONTROL);
2741                 tmp |= DC_HPDx_INT_ACK;
2742                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2743         }
2744         if (ASIC_IS_DCE32(rdev)) {
2745                 if (*disp_int_cont2 & DC_HPD5_INTERRUPT) {
2746                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2747                         tmp |= DC_HPDx_INT_ACK;
2748                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2749                 }
2750                 if (*disp_int_cont2 & DC_HPD6_INTERRUPT) {
2751                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2752                         tmp |= DC_HPDx_INT_ACK;
2753                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2754                 }
2755         }
2756         if (RREG32(R600_HDMI_BLOCK1 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2757                 WREG32_P(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2758         }
2759         if (ASIC_IS_DCE3(rdev)) {
2760                 if (RREG32(R600_HDMI_BLOCK3 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2761                         WREG32_P(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2762                 }
2763         } else {
2764                 if (RREG32(R600_HDMI_BLOCK2 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2765                         WREG32_P(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2766                 }
2767         }
2768 }
2769
2770 void r600_irq_disable(struct radeon_device *rdev)
2771 {
2772         u32 disp_int, disp_int_cont, disp_int_cont2;
2773
2774         r600_disable_interrupts(rdev);
2775         /* Wait and acknowledge irq */
2776         mdelay(1);
2777         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2778         r600_disable_interrupt_state(rdev);
2779 }
2780
2781 static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
2782 {
2783         u32 wptr, tmp;
2784
2785         /* XXX use writeback */
2786         wptr = RREG32(IH_RB_WPTR);
2787
2788         if (wptr & RB_OVERFLOW) {
2789                 /* When a ring buffer overflow happen start parsing interrupt
2790                  * from the last not overwritten vector (wptr + 16). Hopefully
2791                  * this should allow us to catchup.
2792                  */
2793                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2794                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2795                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2796                 tmp = RREG32(IH_RB_CNTL);
2797                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
2798                 WREG32(IH_RB_CNTL, tmp);
2799         }
2800         return (wptr & rdev->ih.ptr_mask);
2801 }
2802
2803 /*        r600 IV Ring
2804  * Each IV ring entry is 128 bits:
2805  * [7:0]    - interrupt source id
2806  * [31:8]   - reserved
2807  * [59:32]  - interrupt source data
2808  * [127:60]  - reserved
2809  *
2810  * The basic interrupt vector entries
2811  * are decoded as follows:
2812  * src_id  src_data  description
2813  *      1         0  D1 Vblank
2814  *      1         1  D1 Vline
2815  *      5         0  D2 Vblank
2816  *      5         1  D2 Vline
2817  *     19         0  FP Hot plug detection A
2818  *     19         1  FP Hot plug detection B
2819  *     19         2  DAC A auto-detection
2820  *     19         3  DAC B auto-detection
2821  *     21         4  HDMI block A
2822  *     21         5  HDMI block B
2823  *    176         -  CP_INT RB
2824  *    177         -  CP_INT IB1
2825  *    178         -  CP_INT IB2
2826  *    181         -  EOP Interrupt
2827  *    233         -  GUI Idle
2828  *
2829  * Note, these are based on r600 and may need to be
2830  * adjusted or added to on newer asics
2831  */
2832
2833 int r600_irq_process(struct radeon_device *rdev)
2834 {
2835         u32 wptr = r600_get_ih_wptr(rdev);
2836         u32 rptr = rdev->ih.rptr;
2837         u32 src_id, src_data;
2838         u32 ring_index, disp_int, disp_int_cont, disp_int_cont2;
2839         unsigned long flags;
2840         bool queue_hotplug = false;
2841
2842         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2843         if (!rdev->ih.enabled)
2844                 return IRQ_NONE;
2845
2846         spin_lock_irqsave(&rdev->ih.lock, flags);
2847
2848         if (rptr == wptr) {
2849                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2850                 return IRQ_NONE;
2851         }
2852         if (rdev->shutdown) {
2853                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2854                 return IRQ_NONE;
2855         }
2856
2857 restart_ih:
2858         /* display interrupts */
2859         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2860
2861         rdev->ih.wptr = wptr;
2862         while (rptr != wptr) {
2863                 /* wptr/rptr are in bytes! */
2864                 ring_index = rptr / 4;
2865                 src_id =  rdev->ih.ring[ring_index] & 0xff;
2866                 src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
2867
2868                 switch (src_id) {
2869                 case 1: /* D1 vblank/vline */
2870                         switch (src_data) {
2871                         case 0: /* D1 vblank */
2872                                 if (disp_int & LB_D1_VBLANK_INTERRUPT) {
2873                                         drm_handle_vblank(rdev->ddev, 0);
2874                                         rdev->pm.vblank_sync = true;
2875                                         wake_up(&rdev->irq.vblank_queue);
2876                                         disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2877                                         DRM_DEBUG("IH: D1 vblank\n");
2878                                 }
2879                                 break;
2880                         case 1: /* D1 vline */
2881                                 if (disp_int & LB_D1_VLINE_INTERRUPT) {
2882                                         disp_int &= ~LB_D1_VLINE_INTERRUPT;
2883                                         DRM_DEBUG("IH: D1 vline\n");
2884                                 }
2885                                 break;
2886                         default:
2887                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2888                                 break;
2889                         }
2890                         break;
2891                 case 5: /* D2 vblank/vline */
2892                         switch (src_data) {
2893                         case 0: /* D2 vblank */
2894                                 if (disp_int & LB_D2_VBLANK_INTERRUPT) {
2895                                         drm_handle_vblank(rdev->ddev, 1);
2896                                         rdev->pm.vblank_sync = true;
2897                                         wake_up(&rdev->irq.vblank_queue);
2898                                         disp_int &= ~LB_D2_VBLANK_INTERRUPT;
2899                                         DRM_DEBUG("IH: D2 vblank\n");
2900                                 }
2901                                 break;
2902                         case 1: /* D1 vline */
2903                                 if (disp_int & LB_D2_VLINE_INTERRUPT) {
2904                                         disp_int &= ~LB_D2_VLINE_INTERRUPT;
2905                                         DRM_DEBUG("IH: D2 vline\n");
2906                                 }
2907                                 break;
2908                         default:
2909                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2910                                 break;
2911                         }
2912                         break;
2913                 case 19: /* HPD/DAC hotplug */
2914                         switch (src_data) {
2915                         case 0:
2916                                 if (disp_int & DC_HPD1_INTERRUPT) {
2917                                         disp_int &= ~DC_HPD1_INTERRUPT;
2918                                         queue_hotplug = true;
2919                                         DRM_DEBUG("IH: HPD1\n");
2920                                 }
2921                                 break;
2922                         case 1:
2923                                 if (disp_int & DC_HPD2_INTERRUPT) {
2924                                         disp_int &= ~DC_HPD2_INTERRUPT;
2925                                         queue_hotplug = true;
2926                                         DRM_DEBUG("IH: HPD2\n");
2927                                 }
2928                                 break;
2929                         case 4:
2930                                 if (disp_int_cont & DC_HPD3_INTERRUPT) {
2931                                         disp_int_cont &= ~DC_HPD3_INTERRUPT;
2932                                         queue_hotplug = true;
2933                                         DRM_DEBUG("IH: HPD3\n");
2934                                 }
2935                                 break;
2936                         case 5:
2937                                 if (disp_int_cont & DC_HPD4_INTERRUPT) {
2938                                         disp_int_cont &= ~DC_HPD4_INTERRUPT;
2939                                         queue_hotplug = true;
2940                                         DRM_DEBUG("IH: HPD4\n");
2941                                 }
2942                                 break;
2943                         case 10:
2944                                 if (disp_int_cont2 & DC_HPD5_INTERRUPT) {
2945                                         disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
2946                                         queue_hotplug = true;
2947                                         DRM_DEBUG("IH: HPD5\n");
2948                                 }
2949                                 break;
2950                         case 12:
2951                                 if (disp_int_cont2 & DC_HPD6_INTERRUPT) {
2952                                         disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
2953                                         queue_hotplug = true;
2954                                         DRM_DEBUG("IH: HPD6\n");
2955                                 }
2956                                 break;
2957                         default:
2958                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2959                                 break;
2960                         }
2961                         break;
2962                 case 21: /* HDMI */
2963                         DRM_DEBUG("IH: HDMI: 0x%x\n", src_data);
2964                         r600_audio_schedule_polling(rdev);
2965                         break;
2966                 case 176: /* CP_INT in ring buffer */
2967                 case 177: /* CP_INT in IB1 */
2968                 case 178: /* CP_INT in IB2 */
2969                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2970                         radeon_fence_process(rdev);
2971                         break;
2972                 case 181: /* CP EOP event */
2973                         DRM_DEBUG("IH: CP EOP\n");
2974                         break;
2975                 case 233: /* GUI IDLE */
2976                         DRM_DEBUG("IH: CP EOP\n");
2977                         rdev->pm.gui_idle = true;
2978                         wake_up(&rdev->irq.idle_queue);
2979                         break;
2980                 default:
2981                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2982                         break;
2983                 }
2984
2985                 /* wptr/rptr are in bytes! */
2986                 rptr += 16;
2987                 rptr &= rdev->ih.ptr_mask;
2988         }
2989         /* make sure wptr hasn't changed while processing */
2990         wptr = r600_get_ih_wptr(rdev);
2991         if (wptr != rdev->ih.wptr)
2992                 goto restart_ih;
2993         if (queue_hotplug)
2994                 queue_work(rdev->wq, &rdev->hotplug_work);
2995         rdev->ih.rptr = rptr;
2996         WREG32(IH_RB_RPTR, rdev->ih.rptr);
2997         spin_unlock_irqrestore(&rdev->ih.lock, flags);
2998         return IRQ_HANDLED;
2999 }
3000
3001 /*
3002  * Debugfs info
3003  */
3004 #if defined(CONFIG_DEBUG_FS)
3005
3006 static int r600_debugfs_cp_ring_info(struct seq_file *m, void *data)
3007 {
3008         struct drm_info_node *node = (struct drm_info_node *) m->private;
3009         struct drm_device *dev = node->minor->dev;
3010         struct radeon_device *rdev = dev->dev_private;
3011         unsigned count, i, j;
3012
3013         radeon_ring_free_size(rdev);
3014         count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
3015         seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
3016         seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
3017         seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
3018         seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
3019         seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
3020         seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
3021         seq_printf(m, "%u dwords in ring\n", count);
3022         i = rdev->cp.rptr;
3023         for (j = 0; j <= count; j++) {
3024                 seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
3025                 i = (i + 1) & rdev->cp.ptr_mask;
3026         }
3027         return 0;
3028 }
3029
3030 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
3031 {
3032         struct drm_info_node *node = (struct drm_info_node *) m->private;
3033         struct drm_device *dev = node->minor->dev;
3034         struct radeon_device *rdev = dev->dev_private;
3035
3036         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
3037         DREG32_SYS(m, rdev, VM_L2_STATUS);
3038         return 0;
3039 }
3040
3041 static struct drm_info_list r600_mc_info_list[] = {
3042         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
3043         {"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
3044 };
3045 #endif
3046
3047 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
3048 {
3049 #if defined(CONFIG_DEBUG_FS)
3050         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
3051 #else
3052         return 0;
3053 #endif
3054 }
3055
3056 /**
3057  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
3058  * rdev: radeon device structure
3059  * bo: buffer object struct which userspace is waiting for idle
3060  *
3061  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
3062  * through ring buffer, this leads to corruption in rendering, see
3063  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
3064  * directly perform HDP flush by writing register through MMIO.
3065  */
3066 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
3067 {
3068         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3069 }