drivers/gpu/drm/r128/r128_cce.c

   1 /* r128_cce.c -- ATI Rage 128 driver -*- linux-c -*-
   2  * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com
   3  */
   4 /*
   5  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   6  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
   7  * All Rights Reserved.
   8  *
   9  * Permission is hereby granted, free of charge, to any person obtaining a
  10  * copy of this software and associated documentation files (the "Software"),
  11  * to deal in the Software without restriction, including without limitation
  12  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13  * and/or sell copies of the Software, and to permit persons to whom the
  14  * Software is furnished to do so, subject to the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the next
  17  * paragraph) shall be included in all copies or substantial portions of the
  18  * Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  23  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  24  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  25  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  26  * DEALINGS IN THE SOFTWARE.
  27  *
  28  * Authors:
  29  *    Gareth Hughes <gareth@valinux.com>
  30  */
  31
  32 #include <linux/firmware.h>
  33 #include <linux/platform_device.h>
  34 #include <linux/slab.h>
  35 #include <linux/module.h>
  36
  37 #include "drmP.h"
  38 #include "drm.h"
  39 #include "r128_drm.h"
  40 #include "r128_drv.h"
  41
  42 #define R128_FIFO_DEBUG         0
  43
  44 #define FIRMWARE_NAME           "r128/r128_cce.bin"
  45
  46 MODULE_FIRMWARE(FIRMWARE_NAME);
  47
  48 static int R128_READ_PLL(struct drm_device *dev, int addr)
  49 {
  50         drm_r128_private_t *dev_priv = dev->dev_private;
  51
  52         R128_WRITE8(R128_CLOCK_CNTL_INDEX, addr & 0x1f);
  53         return R128_READ(R128_CLOCK_CNTL_DATA);
  54 }
  55
  56 #if R128_FIFO_DEBUG
  57 static void r128_status(drm_r128_private_t *dev_priv)
  58 {
  59         printk("GUI_STAT           = 0x%08x\n",
  60                (unsigned int)R128_READ(R128_GUI_STAT));
  61         printk("PM4_STAT           = 0x%08x\n",
  62                (unsigned int)R128_READ(R128_PM4_STAT));
  63         printk("PM4_BUFFER_DL_WPTR = 0x%08x\n",
  64                (unsigned int)R128_READ(R128_PM4_BUFFER_DL_WPTR));
  65         printk("PM4_BUFFER_DL_RPTR = 0x%08x\n",
  66                (unsigned int)R128_READ(R128_PM4_BUFFER_DL_RPTR));
  67         printk("PM4_MICRO_CNTL     = 0x%08x\n",
  68                (unsigned int)R128_READ(R128_PM4_MICRO_CNTL));
  69         printk("PM4_BUFFER_CNTL    = 0x%08x\n",
  70                (unsigned int)R128_READ(R128_PM4_BUFFER_CNTL));
  71 }
  72 #endif
  73
  74 /* ================================================================
  75  * Engine, FIFO control
  76  */
  77
  78 static int r128_do_pixcache_flush(drm_r128_private_t *dev_priv)
  79 {
  80         u32 tmp;
  81         int i;
  82
  83         tmp = R128_READ(R128_PC_NGUI_CTLSTAT) | R128_PC_FLUSH_ALL;
  84         R128_WRITE(R128_PC_NGUI_CTLSTAT, tmp);
  85
  86         for (i = 0; i < dev_priv->usec_timeout; i++) {
  87                 if (!(R128_READ(R128_PC_NGUI_CTLSTAT) & R128_PC_BUSY))
  88                         return 0;
  89                 DRM_UDELAY(1);
  90         }
  91
  92 #if R128_FIFO_DEBUG
  93         DRM_ERROR("failed!\n");
  94 #endif
  95         return -EBUSY;
  96 }
  97
  98 static int r128_do_wait_for_fifo(drm_r128_private_t *dev_priv, int entries)
  99 {
 100         int i;
 101
 102         for (i = 0; i < dev_priv->usec_timeout; i++) {
 103                 int slots = R128_READ(R128_GUI_STAT) & R128_GUI_FIFOCNT_MASK;
 104                 if (slots >= entries)
 105                         return 0;
 106                 DRM_UDELAY(1);
 107         }
 108
 109 #if R128_FIFO_DEBUG
 110         DRM_ERROR("failed!\n");
 111 #endif
 112         return -EBUSY;
 113 }
 114
 115 static int r128_do_wait_for_idle(drm_r128_private_t *dev_priv)
 116 {
 117         int i, ret;
 118
 119         ret = r128_do_wait_for_fifo(dev_priv, 64);
 120         if (ret)
 121                 return ret;
 122
 123         for (i = 0; i < dev_priv->usec_timeout; i++) {
 124                 if (!(R128_READ(R128_GUI_STAT) & R128_GUI_ACTIVE)) {
 125                         r128_do_pixcache_flush(dev_priv);
 126                         return 0;
 127                 }
 128                 DRM_UDELAY(1);
 129         }
 130
 131 #if R128_FIFO_DEBUG
 132         DRM_ERROR("failed!\n");
 133 #endif
 134         return -EBUSY;
 135 }
 136
 137 /* ================================================================
 138  * CCE control, initialization
 139  */
 140
 141 /* Load the microcode for the CCE */
 142 static int r128_cce_load_microcode(drm_r128_private_t *dev_priv)
 143 {
 144         struct platform_device *pdev;
 145         const struct firmware *fw;
 146         const __be32 *fw_data;
 147         int rc, i;
 148
 149         DRM_DEBUG("\n");
 150
 151         pdev = platform_device_register_simple("r128_cce", 0, NULL, 0);
 152         if (IS_ERR(pdev)) {
 153                 printk(KERN_ERR "r128_cce: Failed to register firmware\n");
 154                 return PTR_ERR(pdev);
 155         }
 156         rc = request_firmware(&fw, FIRMWARE_NAME, &pdev->dev);
 157         platform_device_unregister(pdev);
 158         if (rc) {
 159                 printk(KERN_ERR "r128_cce: Failed to load firmware \"%s\"\n",
 160                        FIRMWARE_NAME);
 161                 return rc;
 162         }
 163
 164         if (fw->size != 256 * 8) {
 165                 printk(KERN_ERR
 166                        "r128_cce: Bogus length %zu in firmware \"%s\"\n",
 167                        fw->size, FIRMWARE_NAME);
 168                 rc = -EINVAL;
 169                 goto out_release;
 170         }
 171
 172         r128_do_wait_for_idle(dev_priv);
 173
 174         fw_data = (const __be32 *)fw->data;
 175         R128_WRITE(R128_PM4_MICROCODE_ADDR, 0);
 176         for (i = 0; i < 256; i++) {
 177                 R128_WRITE(R128_PM4_MICROCODE_DATAH,
 178                            be32_to_cpup(&fw_data[i * 2]));
 179                 R128_WRITE(R128_PM4_MICROCODE_DATAL,
 180                            be32_to_cpup(&fw_data[i * 2 + 1]));
 181         }
 182
 183 out_release:
 184         release_firmware(fw);
 185         return rc;
 186 }
 187
 188 /* Flush any pending commands to the CCE.  This should only be used just
 189  * prior to a wait for idle, as it informs the engine that the command
 190  * stream is ending.
 191  */
 192 static void r128_do_cce_flush(drm_r128_private_t *dev_priv)
 193 {
 194         u32 tmp;
 195
 196         tmp = R128_READ(R128_PM4_BUFFER_DL_WPTR) | R128_PM4_BUFFER_DL_DONE;
 197         R128_WRITE(R128_PM4_BUFFER_DL_WPTR, tmp);
 198 }
 199
 200 /* Wait for the CCE to go idle.
 201  */
 202 int r128_do_cce_idle(drm_r128_private_t *dev_priv)
 203 {
 204         int i;
 205
 206         for (i = 0; i < dev_priv->usec_timeout; i++) {
 207                 if (GET_RING_HEAD(dev_priv) == dev_priv->ring.tail) {
 208                         int pm4stat = R128_READ(R128_PM4_STAT);
 209                         if (((pm4stat & R128_PM4_FIFOCNT_MASK) >=
 210                              dev_priv->cce_fifo_size) &&
 211                             !(pm4stat & (R128_PM4_BUSY |
 212                                          R128_PM4_GUI_ACTIVE))) {
 213                                 return r128_do_pixcache_flush(dev_priv);
 214                         }
 215                 }
 216                 DRM_UDELAY(1);
 217         }
 218
 219 #if R128_FIFO_DEBUG
 220         DRM_ERROR("failed!\n");
 221         r128_status(dev_priv);
 222 #endif
 223         return -EBUSY;
 224 }
 225
 226 /* Start the Concurrent Command Engine.
 227  */
 228 static void r128_do_cce_start(drm_r128_private_t *dev_priv)
 229 {
 230         r128_do_wait_for_idle(dev_priv);
 231
 232         R128_WRITE(R128_PM4_BUFFER_CNTL,
 233                    dev_priv->cce_mode | dev_priv->ring.size_l2qw
 234                    | R128_PM4_BUFFER_CNTL_NOUPDATE);
 235         R128_READ(R128_PM4_BUFFER_ADDR);        /* as per the sample code */
 236         R128_WRITE(R128_PM4_MICRO_CNTL, R128_PM4_MICRO_FREERUN);
 237
 238         dev_priv->cce_running = 1;
 239 }
 240
 241 /* Reset the Concurrent Command Engine.  This will not flush any pending
 242  * commands, so you must wait for the CCE command stream to complete
 243  * before calling this routine.
 244  */
 245 static void r128_do_cce_reset(drm_r128_private_t *dev_priv)
 246 {
 247         R128_WRITE(R128_PM4_BUFFER_DL_WPTR, 0);
 248         R128_WRITE(R128_PM4_BUFFER_DL_RPTR, 0);
 249         dev_priv->ring.tail = 0;
 250 }
 251
 252 /* Stop the Concurrent Command Engine.  This will not flush any pending
 253  * commands, so you must flush the command stream and wait for the CCE
 254  * to go idle before calling this routine.
 255  */
 256 static void r128_do_cce_stop(drm_r128_private_t *dev_priv)
 257 {
 258         R128_WRITE(R128_PM4_MICRO_CNTL, 0);
 259         R128_WRITE(R128_PM4_BUFFER_CNTL,
 260                    R128_PM4_NONPM4 | R128_PM4_BUFFER_CNTL_NOUPDATE);
 261
 262         dev_priv->cce_running = 0;
 263 }
 264
 265 /* Reset the engine.  This will stop the CCE if it is running.
 266  */
 267 static int r128_do_engine_reset(struct drm_device *dev)
 268 {
 269         drm_r128_private_t *dev_priv = dev->dev_private;
 270         u32 clock_cntl_index, mclk_cntl, gen_reset_cntl;
 271
 272         r128_do_pixcache_flush(dev_priv);
 273
 274         clock_cntl_index = R128_READ(R128_CLOCK_CNTL_INDEX);
 275         mclk_cntl = R128_READ_PLL(dev, R128_MCLK_CNTL);
 276
 277         R128_WRITE_PLL(R128_MCLK_CNTL,
 278                        mclk_cntl | R128_FORCE_GCP | R128_FORCE_PIPE3D_CP);
 279
 280         gen_reset_cntl = R128_READ(R128_GEN_RESET_CNTL);
 281
 282         /* Taken from the sample code - do not change */
 283         R128_WRITE(R128_GEN_RESET_CNTL, gen_reset_cntl | R128_SOFT_RESET_GUI);
 284         R128_READ(R128_GEN_RESET_CNTL);
 285         R128_WRITE(R128_GEN_RESET_CNTL, gen_reset_cntl & ~R128_SOFT_RESET_GUI);
 286         R128_READ(R128_GEN_RESET_CNTL);
 287
 288         R128_WRITE_PLL(R128_MCLK_CNTL, mclk_cntl);
 289         R128_WRITE(R128_CLOCK_CNTL_INDEX, clock_cntl_index);
 290         R128_WRITE(R128_GEN_RESET_CNTL, gen_reset_cntl);
 291
 292         /* Reset the CCE ring */
 293         r128_do_cce_reset(dev_priv);
 294
 295         /* The CCE is no longer running after an engine reset */
 296         dev_priv->cce_running = 0;
 297
 298         /* Reset any pending vertex, indirect buffers */
 299         r128_freelist_reset(dev);
 300
 301         return 0;
 302 }
 303
 304 static void r128_cce_init_ring_buffer(struct drm_device *dev,
 305                                       drm_r128_private_t *dev_priv)
 306 {
 307         u32 ring_start;
 308         u32 tmp;
 309
 310         DRM_DEBUG("\n");
 311
 312         /* The manual (p. 2) says this address is in "VM space".  This
 313          * means it's an offset from the start of AGP space.
 314          */
 315 #if __OS_HAS_AGP
 316         if (!dev_priv->is_pci)
 317                 ring_start = dev_priv->cce_ring->offset - dev->agp->base;
 318         else
 319 #endif
 320                 ring_start = dev_priv->cce_ring->offset -
 321                     (unsigned long)dev->sg->virtual;
 322
 323         R128_WRITE(R128_PM4_BUFFER_OFFSET, ring_start | R128_AGP_OFFSET);
 324
 325         R128_WRITE(R128_PM4_BUFFER_DL_WPTR, 0);
 326         R128_WRITE(R128_PM4_BUFFER_DL_RPTR, 0);
 327
 328         /* Set watermark control */
 329         R128_WRITE(R128_PM4_BUFFER_WM_CNTL,
 330                    ((R128_WATERMARK_L / 4) << R128_WMA_SHIFT)
 331                    | ((R128_WATERMARK_M / 4) << R128_WMB_SHIFT)
 332                    | ((R128_WATERMARK_N / 4) << R128_WMC_SHIFT)
 333                    | ((R128_WATERMARK_K / 64) << R128_WB_WM_SHIFT));
 334
 335         /* Force read.  Why?  Because it's in the examples... */
 336         R128_READ(R128_PM4_BUFFER_ADDR);
 337
 338         /* Turn on bus mastering */
 339         tmp = R128_READ(R128_BUS_CNTL) & ~R128_BUS_MASTER_DIS;
 340         R128_WRITE(R128_BUS_CNTL, tmp);
 341 }
 342
 343 static int r128_do_init_cce(struct drm_device *dev, drm_r128_init_t *init)
 344 {
 345         drm_r128_private_t *dev_priv;
 346         int rc;
 347
 348         DRM_DEBUG("\n");
 349
 350         if (dev->dev_private) {
 351                 DRM_DEBUG("called when already initialized\n");
 352                 return -EINVAL;
 353         }
 354
 355         dev_priv = kzalloc(sizeof(drm_r128_private_t), GFP_KERNEL);
 356         if (dev_priv == NULL)
 357                 return -ENOMEM;
 358
 359         dev_priv->is_pci = init->is_pci;
 360
 361         if (dev_priv->is_pci && !dev->sg) {
 362                 DRM_ERROR("PCI GART memory not allocated!\n");
 363                 dev->dev_private = (void *)dev_priv;
 364                 r128_do_cleanup_cce(dev);
 365                 return -EINVAL;
 366         }
 367
 368         dev_priv->usec_timeout = init->usec_timeout;
 369         if (dev_priv->usec_timeout < 1 ||
 370             dev_priv->usec_timeout > R128_MAX_USEC_TIMEOUT) {
 371                 DRM_DEBUG("TIMEOUT problem!\n");
 372                 dev->dev_private = (void *)dev_priv;
 373                 r128_do_cleanup_cce(dev);
 374                 return -EINVAL;
 375         }
 376
 377         dev_priv->cce_mode = init->cce_mode;
 378
 379         /* GH: Simple idle check.
 380          */
 381         atomic_set(&dev_priv->idle_count, 0);
 382
 383         /* We don't support anything other than bus-mastering ring mode,
 384          * but the ring can be in either AGP or PCI space for the ring
 385          * read pointer.
 386          */
 387         if ((init->cce_mode != R128_PM4_192BM) &&
 388             (init->cce_mode != R128_PM4_128BM_64INDBM) &&
 389             (init->cce_mode != R128_PM4_64BM_128INDBM) &&
 390             (init->cce_mode != R128_PM4_64BM_64VCBM_64INDBM)) {
 391                 DRM_DEBUG("Bad cce_mode!\n");
 392                 dev->dev_private = (void *)dev_priv;
 393                 r128_do_cleanup_cce(dev);
 394                 return -EINVAL;
 395         }
 396
 397         switch (init->cce_mode) {
 398         case R128_PM4_NONPM4:
 399                 dev_priv->cce_fifo_size = 0;
 400                 break;
 401         case R128_PM4_192PIO:
 402         case R128_PM4_192BM:
 403                 dev_priv->cce_fifo_size = 192;
 404                 break;
 405         case R128_PM4_128PIO_64INDBM:
 406         case R128_PM4_128BM_64INDBM:
 407                 dev_priv->cce_fifo_size = 128;
 408                 break;
 409         case R128_PM4_64PIO_128INDBM:
 410         case R128_PM4_64BM_128INDBM:
 411         case R128_PM4_64PIO_64VCBM_64INDBM:
 412         case R128_PM4_64BM_64VCBM_64INDBM:
 413         case R128_PM4_64PIO_64VCPIO_64INDPIO:
 414                 dev_priv->cce_fifo_size = 64;
 415                 break;
 416         }
 417
 418         switch (init->fb_bpp) {
 419         case 16:
 420                 dev_priv->color_fmt = R128_DATATYPE_RGB565;
 421                 break;
 422         case 32:
 423         default:
 424                 dev_priv->color_fmt = R128_DATATYPE_ARGB8888;
 425                 break;
 426         }
 427         dev_priv->front_offset = init->front_offset;
 428         dev_priv->front_pitch = init->front_pitch;
 429         dev_priv->back_offset = init->back_offset;
 430         dev_priv->back_pitch = init->back_pitch;
 431
 432         switch (init->depth_bpp) {
 433         case 16:
 434                 dev_priv->depth_fmt = R128_DATATYPE_RGB565;
 435                 break;
 436         case 24:
 437         case 32:
 438         default:
 439                 dev_priv->depth_fmt = R128_DATATYPE_ARGB8888;
 440                 break;
 441         }
 442         dev_priv->depth_offset = init->depth_offset;
 443         dev_priv->depth_pitch = init->depth_pitch;
 444         dev_priv->span_offset = init->span_offset;
 445
 446         dev_priv->front_pitch_offset_c = (((dev_priv->front_pitch / 8) << 21) |
 447                                           (dev_priv->front_offset >> 5));
 448         dev_priv->back_pitch_offset_c = (((dev_priv->back_pitch / 8) << 21) |
 449                                          (dev_priv->back_offset >> 5));
 450         dev_priv->depth_pitch_offset_c = (((dev_priv->depth_pitch / 8) << 21) |
 451                                           (dev_priv->depth_offset >> 5) |
 452                                           R128_DST_TILE);
 453         dev_priv->span_pitch_offset_c = (((dev_priv->depth_pitch / 8) << 21) |
 454                                          (dev_priv->span_offset >> 5));
 455
 456         dev_priv->sarea = drm_getsarea(dev);
 457         if (!dev_priv->sarea) {
 458                 DRM_ERROR("could not find sarea!\n");
 459                 dev->dev_private = (void *)dev_priv;
 460                 r128_do_cleanup_cce(dev);
 461                 return -EINVAL;
 462         }
 463
 464         dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
 465         if (!dev_priv->mmio) {
 466                 DRM_ERROR("could not find mmio region!\n");
 467                 dev->dev_private = (void *)dev_priv;
 468                 r128_do_cleanup_cce(dev);
 469                 return -EINVAL;
 470         }
 471         dev_priv->cce_ring = drm_core_findmap(dev, init->ring_offset);
 472         if (!dev_priv->cce_ring) {
 473                 DRM_ERROR("could not find cce ring region!\n");
 474                 dev->dev_private = (void *)dev_priv;
 475                 r128_do_cleanup_cce(dev);
 476                 return -EINVAL;
 477         }
 478         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
 479         if (!dev_priv->ring_rptr) {
 480                 DRM_ERROR("could not find ring read pointer!\n");
 481                 dev->dev_private = (void *)dev_priv;
 482                 r128_do_cleanup_cce(dev);
 483                 return -EINVAL;
 484         }
 485         dev->agp_buffer_token = init->buffers_offset;
 486         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
 487         if (!dev->agp_buffer_map) {
 488                 DRM_ERROR("could not find dma buffer region!\n");
 489                 dev->dev_private = (void *)dev_priv;
 490                 r128_do_cleanup_cce(dev);
 491                 return -EINVAL;
 492         }
 493
 494         if (!dev_priv->is_pci) {
 495                 dev_priv->agp_textures =
 496                     drm_core_findmap(dev, init->agp_textures_offset);
 497                 if (!dev_priv->agp_textures) {
 498                         DRM_ERROR("could not find agp texture region!\n");
 499                         dev->dev_private = (void *)dev_priv;
 500                         r128_do_cleanup_cce(dev);
 501                         return -EINVAL;
 502                 }
 503         }
 504
 505         dev_priv->sarea_priv =
 506             (drm_r128_sarea_t *) ((u8 *) dev_priv->sarea->handle +
 507                                   init->sarea_priv_offset);
 508
 509 #if __OS_HAS_AGP
 510         if (!dev_priv->is_pci) {
 511                 drm_core_ioremap_wc(dev_priv->cce_ring, dev);
 512                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
 513                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
 514                 if (!dev_priv->cce_ring->handle ||
 515                     !dev_priv->ring_rptr->handle ||
 516                     !dev->agp_buffer_map->handle) {
 517                         DRM_ERROR("Could not ioremap agp regions!\n");
 518                         dev->dev_private = (void *)dev_priv;
 519                         r128_do_cleanup_cce(dev);
 520                         return -ENOMEM;
 521                 }
 522         } else
 523 #endif
 524         {
 525                 dev_priv->cce_ring->handle =
 526                         (void *)(unsigned long)dev_priv->cce_ring->offset;
 527                 dev_priv->ring_rptr->handle =
 528                         (void *)(unsigned long)dev_priv->ring_rptr->offset;
 529                 dev->agp_buffer_map->handle =
 530                         (void *)(unsigned long)dev->agp_buffer_map->offset;
 531         }
 532
 533 #if __OS_HAS_AGP
 534         if (!dev_priv->is_pci)
 535                 dev_priv->cce_buffers_offset = dev->agp->base;
 536         else
 537 #endif
 538                 dev_priv->cce_buffers_offset = (unsigned long)dev->sg->virtual;
 539
 540         dev_priv->ring.start = (u32 *) dev_priv->cce_ring->handle;
 541         dev_priv->ring.end = ((u32 *) dev_priv->cce_ring->handle
 542                               + init->ring_size / sizeof(u32));
 543         dev_priv->ring.size = init->ring_size;
 544         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
 545
 546         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
 547
 548         dev_priv->ring.high_mark = 128;
 549
 550         dev_priv->sarea_priv->last_frame = 0;
 551         R128_WRITE(R128_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame);
 552
 553         dev_priv->sarea_priv->last_dispatch = 0;
 554         R128_WRITE(R128_LAST_DISPATCH_REG, dev_priv->sarea_priv->last_dispatch);
 555
 556 #if __OS_HAS_AGP
 557         if (dev_priv->is_pci) {
 558 #endif
 559                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
 560                 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
 561                 dev_priv->gart_info.table_size = R128_PCIGART_TABLE_SIZE;
 562                 dev_priv->gart_info.addr = NULL;
 563                 dev_priv->gart_info.bus_addr = 0;
 564                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
 565                 if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
 566                         DRM_ERROR("failed to init PCI GART!\n");
 567                         dev->dev_private = (void *)dev_priv;
 568                         r128_do_cleanup_cce(dev);
 569                         return -ENOMEM;
 570                 }
 571                 R128_WRITE(R128_PCI_GART_PAGE, dev_priv->gart_info.bus_addr);
 572 #if __OS_HAS_AGP
 573         }
 574 #endif
 575
 576         r128_cce_init_ring_buffer(dev, dev_priv);
 577         rc = r128_cce_load_microcode(dev_priv);
 578
 579         dev->dev_private = (void *)dev_priv;
 580
 581         r128_do_engine_reset(dev);
 582
 583         if (rc) {
 584                 DRM_ERROR("Failed to load firmware!\n");
 585                 r128_do_cleanup_cce(dev);
 586         }
 587
 588         return rc;
 589 }
 590
 591 int r128_do_cleanup_cce(struct drm_device *dev)
 592 {
 593
 594         /* Make sure interrupts are disabled here because the uninstall ioctl
 595          * may not have been called from userspace and after dev_private
 596          * is freed, it's too late.
 597          */
 598         if (dev->irq_enabled)
 599                 drm_irq_uninstall(dev);
 600
 601         if (dev->dev_private) {
 602                 drm_r128_private_t *dev_priv = dev->dev_private;
 603
 604 #if __OS_HAS_AGP
 605                 if (!dev_priv->is_pci) {
 606                         if (dev_priv->cce_ring != NULL)
 607                                 drm_core_ioremapfree(dev_priv->cce_ring, dev);
 608                         if (dev_priv->ring_rptr != NULL)
 609                                 drm_core_ioremapfree(dev_priv->ring_rptr, dev);
 610                         if (dev->agp_buffer_map != NULL) {
 611                                 drm_core_ioremapfree(dev->agp_buffer_map, dev);
 612                                 dev->agp_buffer_map = NULL;
 613                         }
 614                 } else
 615 #endif
 616                 {
 617                         if (dev_priv->gart_info.bus_addr)
 618                                 if (!drm_ati_pcigart_cleanup(dev,
 619                                                         &dev_priv->gart_info))
 620                                         DRM_ERROR
 621                                             ("failed to cleanup PCI GART!\n");
 622                 }
 623
 624                 kfree(dev->dev_private);
 625                 dev->dev_private = NULL;
 626         }
 627
 628         return 0;
 629 }
 630
 631 int r128_cce_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 632 {
 633         drm_r128_init_t *init = data;
 634
 635         DRM_DEBUG("\n");
 636
 637         LOCK_TEST_WITH_RETURN(dev, file_priv);
 638
 639         switch (init->func) {
 640         case R128_INIT_CCE:
 641                 return r128_do_init_cce(dev, init);
 642         case R128_CLEANUP_CCE:
 643                 return r128_do_cleanup_cce(dev);
 644         }
 645
 646         return -EINVAL;
 647 }
 648
 649 int r128_cce_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
 650 {
 651         drm_r128_private_t *dev_priv = dev->dev_private;
 652         DRM_DEBUG("\n");
 653
 654         LOCK_TEST_WITH_RETURN(dev, file_priv);
 655
 656         DEV_INIT_TEST_WITH_RETURN(dev_priv);
 657
 658         if (dev_priv->cce_running || dev_priv->cce_mode == R128_PM4_NONPM4) {
 659                 DRM_DEBUG("while CCE running\n");
 660                 return 0;
 661         }
 662
 663         r128_do_cce_start(dev_priv);
 664
 665         return 0;
 666 }
 667
 668 /* Stop the CCE.  The engine must have been idled before calling this
 669  * routine.
 670  */
 671 int r128_cce_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
 672 {
 673         drm_r128_private_t *dev_priv = dev->dev_private;
 674         drm_r128_cce_stop_t *stop = data;
 675         int ret;
 676         DRM_DEBUG("\n");
 677
 678         LOCK_TEST_WITH_RETURN(dev, file_priv);
 679
 680         DEV_INIT_TEST_WITH_RETURN(dev_priv);
 681
 682         /* Flush any pending CCE commands.  This ensures any outstanding
 683          * commands are exectuted by the engine before we turn it off.
 684          */
 685         if (stop->flush)
 686                 r128_do_cce_flush(dev_priv);
 687
 688         /* If we fail to make the engine go idle, we return an error
 689          * code so that the DRM ioctl wrapper can try again.
 690          */
 691         if (stop->idle) {
 692                 ret = r128_do_cce_idle(dev_priv);
 693                 if (ret)
 694                         return ret;
 695         }
 696
 697         /* Finally, we can turn off the CCE.  If the engine isn't idle,
 698          * we will get some dropped triangles as they won't be fully
 699          * rendered before the CCE is shut down.
 700          */
 701         r128_do_cce_stop(dev_priv);
 702
 703         /* Reset the engine */
 704         r128_do_engine_reset(dev);
 705
 706         return 0;
 707 }
 708
 709 /* Just reset the CCE ring.  Called as part of an X Server engine reset.
 710  */
 711 int r128_cce_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 712 {
 713         drm_r128_private_t *dev_priv = dev->dev_private;
 714         DRM_DEBUG("\n");
 715
 716         LOCK_TEST_WITH_RETURN(dev, file_priv);
 717
 718         DEV_INIT_TEST_WITH_RETURN(dev_priv);
 719
 720         r128_do_cce_reset(dev_priv);
 721
 722         /* The CCE is no longer running after an engine reset */
 723         dev_priv->cce_running = 0;
 724
 725         return 0;
 726 }
 727
 728 int r128_cce_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
 729 {
 730         drm_r128_private_t *dev_priv = dev->dev_private;
 731         DRM_DEBUG("\n");
 732
 733         LOCK_TEST_WITH_RETURN(dev, file_priv);
 734
 735         DEV_INIT_TEST_WITH_RETURN(dev_priv);
 736
 737         if (dev_priv->cce_running)
 738                 r128_do_cce_flush(dev_priv);
 739
 740         return r128_do_cce_idle(dev_priv);
 741 }
 742
 743 int r128_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 744 {
 745         DRM_DEBUG("\n");
 746
 747         LOCK_TEST_WITH_RETURN(dev, file_priv);
 748
 749         DEV_INIT_TEST_WITH_RETURN(dev->dev_private);
 750
 751         return r128_do_engine_reset(dev);
 752 }
 753
 754 int r128_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
 755 {
 756         return -EINVAL;
 757 }
 758
 759 /* ================================================================
 760  * Freelist management
 761  */
 762 #define R128_BUFFER_USED        0xffffffff
 763 #define R128_BUFFER_FREE        0
 764
 765 #if 0
 766 static int r128_freelist_init(struct drm_device *dev)
 767 {
 768         struct drm_device_dma *dma = dev->dma;
 769         drm_r128_private_t *dev_priv = dev->dev_private;
 770         struct drm_buf *buf;
 771         drm_r128_buf_priv_t *buf_priv;
 772         drm_r128_freelist_t *entry;
 773         int i;
 774
 775         dev_priv->head = kzalloc(sizeof(drm_r128_freelist_t), GFP_KERNEL);
 776         if (dev_priv->head == NULL)
 777                 return -ENOMEM;
 778
 779         dev_priv->head->age = R128_BUFFER_USED;
 780
 781         for (i = 0; i < dma->buf_count; i++) {
 782                 buf = dma->buflist[i];
 783                 buf_priv = buf->dev_private;
 784
 785                 entry = kmalloc(sizeof(drm_r128_freelist_t), GFP_KERNEL);
 786                 if (!entry)
 787                         return -ENOMEM;
 788
 789                 entry->age = R128_BUFFER_FREE;
 790                 entry->buf = buf;
 791                 entry->prev = dev_priv->head;
 792                 entry->next = dev_priv->head->next;
 793                 if (!entry->next)
 794                         dev_priv->tail = entry;
 795
 796                 buf_priv->discard = 0;
 797                 buf_priv->dispatched = 0;
 798                 buf_priv->list_entry = entry;
 799
 800                 dev_priv->head->next = entry;
 801
 802                 if (dev_priv->head->next)
 803                         dev_priv->head->next->prev = entry;
 804         }
 805
 806         return 0;
 807
 808 }
 809 #endif
 810
 811 static struct drm_buf *r128_freelist_get(struct drm_device * dev)
 812 {
 813         struct drm_device_dma *dma = dev->dma;
 814         drm_r128_private_t *dev_priv = dev->dev_private;
 815         drm_r128_buf_priv_t *buf_priv;
 816         struct drm_buf *buf;
 817         int i, t;
 818
 819         /* FIXME: Optimize -- use freelist code */
 820
 821         for (i = 0; i < dma->buf_count; i++) {
 822                 buf = dma->buflist[i];
 823                 buf_priv = buf->dev_private;
 824                 if (!buf->file_priv)
 825                         return buf;
 826         }
 827
 828         for (t = 0; t < dev_priv->usec_timeout; t++) {
 829                 u32 done_age = R128_READ(R128_LAST_DISPATCH_REG);
 830
 831                 for (i = 0; i < dma->buf_count; i++) {
 832                         buf = dma->buflist[i];
 833                         buf_priv = buf->dev_private;
 834                         if (buf->pending && buf_priv->age <= done_age) {
 835                                 /* The buffer has been processed, so it
 836                                  * can now be used.
 837                                  */
 838                                 buf->pending = 0;
 839                                 return buf;
 840                         }
 841                 }
 842                 DRM_UDELAY(1);
 843         }
 844
 845         DRM_DEBUG("returning NULL!\n");
 846         return NULL;
 847 }
 848
 849 void r128_freelist_reset(struct drm_device *dev)
 850 {
 851         struct drm_device_dma *dma = dev->dma;
 852         int i;
 853
 854         for (i = 0; i < dma->buf_count; i++) {
 855                 struct drm_buf *buf = dma->buflist[i];
 856                 drm_r128_buf_priv_t *buf_priv = buf->dev_private;
 857                 buf_priv->age = 0;
 858         }
 859 }
 860
 861 /* ================================================================
 862  * CCE command submission
 863  */
 864
 865 int r128_wait_ring(drm_r128_private_t *dev_priv, int n)
 866 {
 867         drm_r128_ring_buffer_t *ring = &dev_priv->ring;
 868         int i;
 869
 870         for (i = 0; i < dev_priv->usec_timeout; i++) {
 871                 r128_update_ring_snapshot(dev_priv);
 872                 if (ring->space >= n)
 873                         return 0;
 874                 DRM_UDELAY(1);
 875         }
 876
 877         /* FIXME: This is being ignored... */
 878         DRM_ERROR("failed!\n");
 879         return -EBUSY;
 880 }
 881
 882 static int r128_cce_get_buffers(struct drm_device *dev,
 883                                 struct drm_file *file_priv,
 884                                 struct drm_dma *d)
 885 {
 886         int i;
 887         struct drm_buf *buf;
 888
 889         for (i = d->granted_count; i < d->request_count; i++) {
 890                 buf = r128_freelist_get(dev);
 891                 if (!buf)
 892                         return -EAGAIN;
 893
 894                 buf->file_priv = file_priv;
 895
 896                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
 897                                      sizeof(buf->idx)))
 898                         return -EFAULT;
 899                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
 900                                      sizeof(buf->total)))
 901                         return -EFAULT;
 902
 903                 d->granted_count++;
 904         }
 905         return 0;
 906 }
 907
 908 int r128_cce_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
 909 {
 910         struct drm_device_dma *dma = dev->dma;
 911         int ret = 0;
 912         struct drm_dma *d = data;
 913
 914         LOCK_TEST_WITH_RETURN(dev, file_priv);
 915
 916         /* Please don't send us buffers.
 917          */
 918         if (d->send_count != 0) {
 919                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
 920                           DRM_CURRENTPID, d->send_count);
 921                 return -EINVAL;
 922         }
 923
 924         /* We'll send you buffers.
 925          */
 926         if (d->request_count < 0 || d->request_count > dma->buf_count) {
 927                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
 928                           DRM_CURRENTPID, d->request_count, dma->buf_count);
 929                 return -EINVAL;
 930         }
 931
 932         d->granted_count = 0;
 933
 934         if (d->request_count)
 935                 ret = r128_cce_get_buffers(dev, file_priv, d);
 936
 937         return ret;
 938 }