drivers/char/drm/radeon_state.c

   1 /* radeon_state.c -- State support for Radeon -*- linux-c -*- */
   2 /*
   3  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  21  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  22  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23  * DEALINGS IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *    Gareth Hughes <gareth@valinux.com>
  27  *    Kevin E. Martin <martin@valinux.com>
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "drm_sarea.h"
  33 #include "radeon_drm.h"
  34 #include "radeon_drv.h"
  35
  36 /* ================================================================
  37  * Helper functions for client state checking and fixup
  38  */
  39
  40 static __inline__ int radeon_check_and_fixup_offset(drm_radeon_private_t *
  41                                                     dev_priv,
  42                                                     drm_file_t * filp_priv,
  43                                                     u32 *offset)
  44 {
  45         u64 off = *offset;
  46         u32 fb_end = dev_priv->fb_location + dev_priv->fb_size - 1;
  47         struct drm_radeon_driver_file_fields *radeon_priv;
  48
  49         /* Hrm ... the story of the offset ... So this function converts
  50          * the various ideas of what userland clients might have for an
  51          * offset in the card address space into an offset into the card
  52          * address space :) So with a sane client, it should just keep
  53          * the value intact and just do some boundary checking. However,
  54          * not all clients are sane. Some older clients pass us 0 based
  55          * offsets relative to the start of the framebuffer and some may
  56          * assume the AGP aperture it appended to the framebuffer, so we
  57          * try to detect those cases and fix them up.
  58          *
  59          * Note: It might be a good idea here to make sure the offset lands
  60          * in some "allowed" area to protect things like the PCIE GART...
  61          */
  62
  63         /* First, the best case, the offset already lands in either the
  64          * framebuffer or the GART mapped space
  65          */
  66         if (radeon_check_offset(dev_priv, off))
  67                 return 0;
  68
  69         /* Ok, that didn't happen... now check if we have a zero based
  70          * offset that fits in the framebuffer + gart space, apply the
  71          * magic offset we get from SETPARAM or calculated from fb_location
  72          */
  73         if (off < (dev_priv->fb_size + dev_priv->gart_size)) {
  74                 radeon_priv = filp_priv->driver_priv;
  75                 off += radeon_priv->radeon_fb_delta;
  76         }
  77
  78         /* Finally, assume we aimed at a GART offset if beyond the fb */
  79         if (off > fb_end)
  80                 off = off - fb_end - 1 + dev_priv->gart_vm_start;
  81
  82         /* Now recheck and fail if out of bounds */
  83         if (radeon_check_offset(dev_priv, off)) {
  84                 DRM_DEBUG("offset fixed up to 0x%x\n", (unsigned int)off);
  85                 *offset = off;
  86                 return 0;
  87         }
  88         return DRM_ERR(EINVAL);
  89 }
  90
  91 static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
  92                                                      dev_priv,
  93                                                      drm_file_t * filp_priv,
  94                                                      int id, u32 *data)
  95 {
  96         switch (id) {
  97
  98         case RADEON_EMIT_PP_MISC:
  99                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 100                     &data[(RADEON_RB3D_DEPTHOFFSET - RADEON_PP_MISC) / 4])) {
 101                         DRM_ERROR("Invalid depth buffer offset\n");
 102                         return DRM_ERR(EINVAL);
 103                 }
 104                 break;
 105
 106         case RADEON_EMIT_PP_CNTL:
 107                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 108                     &data[(RADEON_RB3D_COLOROFFSET - RADEON_PP_CNTL) / 4])) {
 109                         DRM_ERROR("Invalid colour buffer offset\n");
 110                         return DRM_ERR(EINVAL);
 111                 }
 112                 break;
 113
 114         case R200_EMIT_PP_TXOFFSET_0:
 115         case R200_EMIT_PP_TXOFFSET_1:
 116         case R200_EMIT_PP_TXOFFSET_2:
 117         case R200_EMIT_PP_TXOFFSET_3:
 118         case R200_EMIT_PP_TXOFFSET_4:
 119         case R200_EMIT_PP_TXOFFSET_5:
 120                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 121                                                   &data[0])) {
 122                         DRM_ERROR("Invalid R200 texture offset\n");
 123                         return DRM_ERR(EINVAL);
 124                 }
 125                 break;
 126
 127         case RADEON_EMIT_PP_TXFILTER_0:
 128         case RADEON_EMIT_PP_TXFILTER_1:
 129         case RADEON_EMIT_PP_TXFILTER_2:
 130                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 131                     &data[(RADEON_PP_TXOFFSET_0 - RADEON_PP_TXFILTER_0) / 4])) {
 132                         DRM_ERROR("Invalid R100 texture offset\n");
 133                         return DRM_ERR(EINVAL);
 134                 }
 135                 break;
 136
 137         case R200_EMIT_PP_CUBIC_OFFSETS_0:
 138         case R200_EMIT_PP_CUBIC_OFFSETS_1:
 139         case R200_EMIT_PP_CUBIC_OFFSETS_2:
 140         case R200_EMIT_PP_CUBIC_OFFSETS_3:
 141         case R200_EMIT_PP_CUBIC_OFFSETS_4:
 142         case R200_EMIT_PP_CUBIC_OFFSETS_5:{
 143                         int i;
 144                         for (i = 0; i < 5; i++) {
 145                                 if (radeon_check_and_fixup_offset(dev_priv,
 146                                                                   filp_priv,
 147                                                                   &data[i])) {
 148                                         DRM_ERROR
 149                                             ("Invalid R200 cubic texture offset\n");
 150                                         return DRM_ERR(EINVAL);
 151                                 }
 152                         }
 153                         break;
 154                 }
 155
 156         case RADEON_EMIT_PP_CUBIC_OFFSETS_T0:
 157         case RADEON_EMIT_PP_CUBIC_OFFSETS_T1:
 158         case RADEON_EMIT_PP_CUBIC_OFFSETS_T2:{
 159                         int i;
 160                         for (i = 0; i < 5; i++) {
 161                                 if (radeon_check_and_fixup_offset(dev_priv,
 162                                                                   filp_priv,
 163                                                                   &data[i])) {
 164                                         DRM_ERROR
 165                                             ("Invalid R100 cubic texture offset\n");
 166                                         return DRM_ERR(EINVAL);
 167                                 }
 168                         }
 169                 }
 170                 break;
 171
 172         case R200_EMIT_VAP_CTL:{
 173                         RING_LOCALS;
 174                         BEGIN_RING(2);
 175                         OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
 176                         ADVANCE_RING();
 177                 }
 178                 break;
 179
 180         case RADEON_EMIT_RB3D_COLORPITCH:
 181         case RADEON_EMIT_RE_LINE_PATTERN:
 182         case RADEON_EMIT_SE_LINE_WIDTH:
 183         case RADEON_EMIT_PP_LUM_MATRIX:
 184         case RADEON_EMIT_PP_ROT_MATRIX_0:
 185         case RADEON_EMIT_RB3D_STENCILREFMASK:
 186         case RADEON_EMIT_SE_VPORT_XSCALE:
 187         case RADEON_EMIT_SE_CNTL:
 188         case RADEON_EMIT_SE_CNTL_STATUS:
 189         case RADEON_EMIT_RE_MISC:
 190         case RADEON_EMIT_PP_BORDER_COLOR_0:
 191         case RADEON_EMIT_PP_BORDER_COLOR_1:
 192         case RADEON_EMIT_PP_BORDER_COLOR_2:
 193         case RADEON_EMIT_SE_ZBIAS_FACTOR:
 194         case RADEON_EMIT_SE_TCL_OUTPUT_VTX_FMT:
 195         case RADEON_EMIT_SE_TCL_MATERIAL_EMMISSIVE_RED:
 196         case R200_EMIT_PP_TXCBLEND_0:
 197         case R200_EMIT_PP_TXCBLEND_1:
 198         case R200_EMIT_PP_TXCBLEND_2:
 199         case R200_EMIT_PP_TXCBLEND_3:
 200         case R200_EMIT_PP_TXCBLEND_4:
 201         case R200_EMIT_PP_TXCBLEND_5:
 202         case R200_EMIT_PP_TXCBLEND_6:
 203         case R200_EMIT_PP_TXCBLEND_7:
 204         case R200_EMIT_TCL_LIGHT_MODEL_CTL_0:
 205         case R200_EMIT_TFACTOR_0:
 206         case R200_EMIT_VTX_FMT_0:
 207         case R200_EMIT_MATRIX_SELECT_0:
 208         case R200_EMIT_TEX_PROC_CTL_2:
 209         case R200_EMIT_TCL_UCP_VERT_BLEND_CTL:
 210         case R200_EMIT_PP_TXFILTER_0:
 211         case R200_EMIT_PP_TXFILTER_1:
 212         case R200_EMIT_PP_TXFILTER_2:
 213         case R200_EMIT_PP_TXFILTER_3:
 214         case R200_EMIT_PP_TXFILTER_4:
 215         case R200_EMIT_PP_TXFILTER_5:
 216         case R200_EMIT_VTE_CNTL:
 217         case R200_EMIT_OUTPUT_VTX_COMP_SEL:
 218         case R200_EMIT_PP_TAM_DEBUG3:
 219         case R200_EMIT_PP_CNTL_X:
 220         case R200_EMIT_RB3D_DEPTHXY_OFFSET:
 221         case R200_EMIT_RE_AUX_SCISSOR_CNTL:
 222         case R200_EMIT_RE_SCISSOR_TL_0:
 223         case R200_EMIT_RE_SCISSOR_TL_1:
 224         case R200_EMIT_RE_SCISSOR_TL_2:
 225         case R200_EMIT_SE_VAP_CNTL_STATUS:
 226         case R200_EMIT_SE_VTX_STATE_CNTL:
 227         case R200_EMIT_RE_POINTSIZE:
 228         case R200_EMIT_TCL_INPUT_VTX_VECTOR_ADDR_0:
 229         case R200_EMIT_PP_CUBIC_FACES_0:
 230         case R200_EMIT_PP_CUBIC_FACES_1:
 231         case R200_EMIT_PP_CUBIC_FACES_2:
 232         case R200_EMIT_PP_CUBIC_FACES_3:
 233         case R200_EMIT_PP_CUBIC_FACES_4:
 234         case R200_EMIT_PP_CUBIC_FACES_5:
 235         case RADEON_EMIT_PP_TEX_SIZE_0:
 236         case RADEON_EMIT_PP_TEX_SIZE_1:
 237         case RADEON_EMIT_PP_TEX_SIZE_2:
 238         case R200_EMIT_RB3D_BLENDCOLOR:
 239         case R200_EMIT_TCL_POINT_SPRITE_CNTL:
 240         case RADEON_EMIT_PP_CUBIC_FACES_0:
 241         case RADEON_EMIT_PP_CUBIC_FACES_1:
 242         case RADEON_EMIT_PP_CUBIC_FACES_2:
 243         case R200_EMIT_PP_TRI_PERF_CNTL:
 244         case R200_EMIT_PP_AFS_0:
 245         case R200_EMIT_PP_AFS_1:
 246         case R200_EMIT_ATF_TFACTOR:
 247         case R200_EMIT_PP_TXCTLALL_0:
 248         case R200_EMIT_PP_TXCTLALL_1:
 249         case R200_EMIT_PP_TXCTLALL_2:
 250         case R200_EMIT_PP_TXCTLALL_3:
 251         case R200_EMIT_PP_TXCTLALL_4:
 252         case R200_EMIT_PP_TXCTLALL_5:
 253         case R200_EMIT_VAP_PVS_CNTL:
 254                 /* These packets don't contain memory offsets */
 255                 break;
 256
 257         default:
 258                 DRM_ERROR("Unknown state packet ID %d\n", id);
 259                 return DRM_ERR(EINVAL);
 260         }
 261
 262         return 0;
 263 }
 264
 265 static __inline__ int radeon_check_and_fixup_packet3(drm_radeon_private_t *
 266                                                      dev_priv,
 267                                                      drm_file_t *filp_priv,
 268                                                      drm_radeon_kcmd_buffer_t *
 269                                                      cmdbuf,
 270                                                      unsigned int *cmdsz)
 271 {
 272         u32 *cmd = (u32 *) cmdbuf->buf;
 273         u32 offset, narrays;
 274         int count, i, k;
 275
 276         *cmdsz = 2 + ((cmd[0] & RADEON_CP_PACKET_COUNT_MASK) >> 16);
 277
 278         if ((cmd[0] & 0xc0000000) != RADEON_CP_PACKET3) {
 279                 DRM_ERROR("Not a type 3 packet\n");
 280                 return DRM_ERR(EINVAL);
 281         }
 282
 283         if (4 * *cmdsz > cmdbuf->bufsz) {
 284                 DRM_ERROR("Packet size larger than size of data provided\n");
 285                 return DRM_ERR(EINVAL);
 286         }
 287
 288         switch(cmd[0] & 0xff00) {
 289         /* XXX Are there old drivers needing other packets? */
 290
 291         case RADEON_3D_DRAW_IMMD:
 292         case RADEON_3D_DRAW_VBUF:
 293         case RADEON_3D_DRAW_INDX:
 294         case RADEON_WAIT_FOR_IDLE:
 295         case RADEON_CP_NOP:
 296         case RADEON_3D_CLEAR_ZMASK:
 297 /*      case RADEON_CP_NEXT_CHAR:
 298         case RADEON_CP_PLY_NEXTSCAN:
 299         case RADEON_CP_SET_SCISSORS: */ /* probably safe but will never need them? */
 300                 /* these packets are safe */
 301                 break;
 302
 303         case RADEON_CP_3D_DRAW_IMMD_2:
 304         case RADEON_CP_3D_DRAW_VBUF_2:
 305         case RADEON_CP_3D_DRAW_INDX_2:
 306         case RADEON_3D_CLEAR_HIZ:
 307                 /* safe but r200 only */
 308                 if (dev_priv->microcode_version != UCODE_R200) {
 309                         DRM_ERROR("Invalid 3d packet for r100-class chip\n");
 310                         return DRM_ERR(EINVAL);
 311                 }
 312                 break;
 313
 314         case RADEON_3D_LOAD_VBPNTR:
 315                 count = (cmd[0] >> 16) & 0x3fff;
 316
 317                 if (count > 18) { /* 12 arrays max */
 318                         DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
 319                                   count);
 320                         return DRM_ERR(EINVAL);
 321                 }
 322
 323                 /* carefully check packet contents */
 324                 narrays = cmd[1] & ~0xc000;
 325                 k = 0;
 326                 i = 2;
 327                 while ((k < narrays) && (i < (count + 2))) {
 328                         i++;            /* skip attribute field */
 329                         if (radeon_check_and_fixup_offset(dev_priv, filp_priv, &cmd[i])) {
 330                                 DRM_ERROR
 331                                     ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
 332                                      k, i);
 333                                 return DRM_ERR(EINVAL);
 334                         }
 335                         k++;
 336                         i++;
 337                         if (k == narrays)
 338                                 break;
 339                         /* have one more to process, they come in pairs */
 340                         if (radeon_check_and_fixup_offset(dev_priv, filp_priv, &cmd[i])) {
 341                                 DRM_ERROR
 342                                     ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
 343                                      k, i);
 344                                 return DRM_ERR(EINVAL);
 345                         }
 346                         k++;
 347                         i++;
 348                 }
 349                 /* do the counts match what we expect ? */
 350                 if ((k != narrays) || (i != (count + 2))) {
 351                         DRM_ERROR
 352                             ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
 353                               k, i, narrays, count + 1);
 354                         return DRM_ERR(EINVAL);
 355                 }
 356                 break;
 357
 358         case RADEON_3D_RNDR_GEN_INDX_PRIM:
 359                 if (dev_priv->microcode_version != UCODE_R100) {
 360                         DRM_ERROR("Invalid 3d packet for r200-class chip\n");
 361                         return DRM_ERR(EINVAL);
 362                 }
 363                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv, &cmd[1])) {
 364                                 DRM_ERROR("Invalid rndr_gen_indx offset\n");
 365                                 return DRM_ERR(EINVAL);
 366                 }
 367                 break;
 368
 369         case RADEON_CP_INDX_BUFFER:
 370                 if (dev_priv->microcode_version != UCODE_R200) {
 371                         DRM_ERROR("Invalid 3d packet for r100-class chip\n");
 372                         return DRM_ERR(EINVAL);
 373                 }
 374                 if ((cmd[1] & 0x8000ffff) != 0x80000810) {
 375                         DRM_ERROR("Invalid indx_buffer reg address %08X\n", cmd[1]);
 376                         return DRM_ERR(EINVAL);
 377                 }
 378                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv, &cmd[2])) {
 379                         DRM_ERROR("Invalid indx_buffer offset is %08X\n", cmd[2]);
 380                         return DRM_ERR(EINVAL);
 381                 }
 382                 break;
 383
 384         case RADEON_CNTL_HOSTDATA_BLT:
 385         case RADEON_CNTL_PAINT_MULTI:
 386         case RADEON_CNTL_BITBLT_MULTI:
 387                 /* MSB of opcode: next DWORD GUI_CNTL */
 388                 if (cmd[1] & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
 389                               | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 390                         offset = cmd[2] << 10;
 391                         if (radeon_check_and_fixup_offset
 392                             (dev_priv, filp_priv, &offset)) {
 393                                 DRM_ERROR("Invalid first packet offset\n");
 394                                 return DRM_ERR(EINVAL);
 395                         }
 396                         cmd[2] = (cmd[2] & 0xffc00000) | offset >> 10;
 397                 }
 398
 399                 if ((cmd[1] & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) &&
 400                     (cmd[1] & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 401                         offset = cmd[3] << 10;
 402                         if (radeon_check_and_fixup_offset
 403                             (dev_priv, filp_priv, &offset)) {
 404                                 DRM_ERROR("Invalid second packet offset\n");
 405                                 return DRM_ERR(EINVAL);
 406                         }
 407                         cmd[3] = (cmd[3] & 0xffc00000) | offset >> 10;
 408                 }
 409                 break;
 410
 411         default:
 412                 DRM_ERROR("Invalid packet type %x\n", cmd[0] & 0xff00);
 413                 return DRM_ERR(EINVAL);
 414         }
 415
 416         return 0;
 417 }
 418
 419 /* ================================================================
 420  * CP hardware state programming functions
 421  */
 422
 423 static __inline__ void radeon_emit_clip_rect(drm_radeon_private_t * dev_priv,
 424                                              drm_clip_rect_t * box)
 425 {
 426         RING_LOCALS;
 427
 428         DRM_DEBUG("   box:  x1=%d y1=%d  x2=%d y2=%d\n",
 429                   box->x1, box->y1, box->x2, box->y2);
 430
 431         BEGIN_RING(4);
 432         OUT_RING(CP_PACKET0(RADEON_RE_TOP_LEFT, 0));
 433         OUT_RING((box->y1 << 16) | box->x1);
 434         OUT_RING(CP_PACKET0(RADEON_RE_WIDTH_HEIGHT, 0));
 435         OUT_RING(((box->y2 - 1) << 16) | (box->x2 - 1));
 436         ADVANCE_RING();
 437 }
 438
 439 /* Emit 1.1 state
 440  */
 441 static int radeon_emit_state(drm_radeon_private_t * dev_priv,
 442                              drm_file_t * filp_priv,
 443                              drm_radeon_context_regs_t * ctx,
 444                              drm_radeon_texture_regs_t * tex,
 445                              unsigned int dirty)
 446 {
 447         RING_LOCALS;
 448         DRM_DEBUG("dirty=0x%08x\n", dirty);
 449
 450         if (dirty & RADEON_UPLOAD_CONTEXT) {
 451                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 452                                                   &ctx->rb3d_depthoffset)) {
 453                         DRM_ERROR("Invalid depth buffer offset\n");
 454                         return DRM_ERR(EINVAL);
 455                 }
 456
 457                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 458                                                   &ctx->rb3d_coloroffset)) {
 459                         DRM_ERROR("Invalid depth buffer offset\n");
 460                         return DRM_ERR(EINVAL);
 461                 }
 462
 463                 BEGIN_RING(14);
 464                 OUT_RING(CP_PACKET0(RADEON_PP_MISC, 6));
 465                 OUT_RING(ctx->pp_misc);
 466                 OUT_RING(ctx->pp_fog_color);
 467                 OUT_RING(ctx->re_solid_color);
 468                 OUT_RING(ctx->rb3d_blendcntl);
 469                 OUT_RING(ctx->rb3d_depthoffset);
 470                 OUT_RING(ctx->rb3d_depthpitch);
 471                 OUT_RING(ctx->rb3d_zstencilcntl);
 472                 OUT_RING(CP_PACKET0(RADEON_PP_CNTL, 2));
 473                 OUT_RING(ctx->pp_cntl);
 474                 OUT_RING(ctx->rb3d_cntl);
 475                 OUT_RING(ctx->rb3d_coloroffset);
 476                 OUT_RING(CP_PACKET0(RADEON_RB3D_COLORPITCH, 0));
 477                 OUT_RING(ctx->rb3d_colorpitch);
 478                 ADVANCE_RING();
 479         }
 480
 481         if (dirty & RADEON_UPLOAD_VERTFMT) {
 482                 BEGIN_RING(2);
 483                 OUT_RING(CP_PACKET0(RADEON_SE_COORD_FMT, 0));
 484                 OUT_RING(ctx->se_coord_fmt);
 485                 ADVANCE_RING();
 486         }
 487
 488         if (dirty & RADEON_UPLOAD_LINE) {
 489                 BEGIN_RING(5);
 490                 OUT_RING(CP_PACKET0(RADEON_RE_LINE_PATTERN, 1));
 491                 OUT_RING(ctx->re_line_pattern);
 492                 OUT_RING(ctx->re_line_state);
 493                 OUT_RING(CP_PACKET0(RADEON_SE_LINE_WIDTH, 0));
 494                 OUT_RING(ctx->se_line_width);
 495                 ADVANCE_RING();
 496         }
 497
 498         if (dirty & RADEON_UPLOAD_BUMPMAP) {
 499                 BEGIN_RING(5);
 500                 OUT_RING(CP_PACKET0(RADEON_PP_LUM_MATRIX, 0));
 501                 OUT_RING(ctx->pp_lum_matrix);
 502                 OUT_RING(CP_PACKET0(RADEON_PP_ROT_MATRIX_0, 1));
 503                 OUT_RING(ctx->pp_rot_matrix_0);
 504                 OUT_RING(ctx->pp_rot_matrix_1);
 505                 ADVANCE_RING();
 506         }
 507
 508         if (dirty & RADEON_UPLOAD_MASKS) {
 509                 BEGIN_RING(4);
 510                 OUT_RING(CP_PACKET0(RADEON_RB3D_STENCILREFMASK, 2));
 511                 OUT_RING(ctx->rb3d_stencilrefmask);
 512                 OUT_RING(ctx->rb3d_ropcntl);
 513                 OUT_RING(ctx->rb3d_planemask);
 514                 ADVANCE_RING();
 515         }
 516
 517         if (dirty & RADEON_UPLOAD_VIEWPORT) {
 518                 BEGIN_RING(7);
 519                 OUT_RING(CP_PACKET0(RADEON_SE_VPORT_XSCALE, 5));
 520                 OUT_RING(ctx->se_vport_xscale);
 521                 OUT_RING(ctx->se_vport_xoffset);
 522                 OUT_RING(ctx->se_vport_yscale);
 523                 OUT_RING(ctx->se_vport_yoffset);
 524                 OUT_RING(ctx->se_vport_zscale);
 525                 OUT_RING(ctx->se_vport_zoffset);
 526                 ADVANCE_RING();
 527         }
 528
 529         if (dirty & RADEON_UPLOAD_SETUP) {
 530                 BEGIN_RING(4);
 531                 OUT_RING(CP_PACKET0(RADEON_SE_CNTL, 0));
 532                 OUT_RING(ctx->se_cntl);
 533                 OUT_RING(CP_PACKET0(RADEON_SE_CNTL_STATUS, 0));
 534                 OUT_RING(ctx->se_cntl_status);
 535                 ADVANCE_RING();
 536         }
 537
 538         if (dirty & RADEON_UPLOAD_MISC) {
 539                 BEGIN_RING(2);
 540                 OUT_RING(CP_PACKET0(RADEON_RE_MISC, 0));
 541                 OUT_RING(ctx->re_misc);
 542                 ADVANCE_RING();
 543         }
 544
 545         if (dirty & RADEON_UPLOAD_TEX0) {
 546                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 547                                                   &tex[0].pp_txoffset)) {
 548                         DRM_ERROR("Invalid texture offset for unit 0\n");
 549                         return DRM_ERR(EINVAL);
 550                 }
 551
 552                 BEGIN_RING(9);
 553                 OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_0, 5));
 554                 OUT_RING(tex[0].pp_txfilter);
 555                 OUT_RING(tex[0].pp_txformat);
 556                 OUT_RING(tex[0].pp_txoffset);
 557                 OUT_RING(tex[0].pp_txcblend);
 558                 OUT_RING(tex[0].pp_txablend);
 559                 OUT_RING(tex[0].pp_tfactor);
 560                 OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_0, 0));
 561                 OUT_RING(tex[0].pp_border_color);
 562                 ADVANCE_RING();
 563         }
 564
 565         if (dirty & RADEON_UPLOAD_TEX1) {
 566                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 567                                                   &tex[1].pp_txoffset)) {
 568                         DRM_ERROR("Invalid texture offset for unit 1\n");
 569                         return DRM_ERR(EINVAL);
 570                 }
 571
 572                 BEGIN_RING(9);
 573                 OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_1, 5));
 574                 OUT_RING(tex[1].pp_txfilter);
 575                 OUT_RING(tex[1].pp_txformat);
 576                 OUT_RING(tex[1].pp_txoffset);
 577                 OUT_RING(tex[1].pp_txcblend);
 578                 OUT_RING(tex[1].pp_txablend);
 579                 OUT_RING(tex[1].pp_tfactor);
 580                 OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_1, 0));
 581                 OUT_RING(tex[1].pp_border_color);
 582                 ADVANCE_RING();
 583         }
 584
 585         if (dirty & RADEON_UPLOAD_TEX2) {
 586                 if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
 587                                                   &tex[2].pp_txoffset)) {
 588                         DRM_ERROR("Invalid texture offset for unit 2\n");
 589                         return DRM_ERR(EINVAL);
 590                 }
 591
 592                 BEGIN_RING(9);
 593                 OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_2, 5));
 594                 OUT_RING(tex[2].pp_txfilter);
 595                 OUT_RING(tex[2].pp_txformat);
 596                 OUT_RING(tex[2].pp_txoffset);
 597                 OUT_RING(tex[2].pp_txcblend);
 598                 OUT_RING(tex[2].pp_txablend);
 599                 OUT_RING(tex[2].pp_tfactor);
 600                 OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_2, 0));
 601                 OUT_RING(tex[2].pp_border_color);
 602                 ADVANCE_RING();
 603         }
 604
 605         return 0;
 606 }
 607
 608 /* Emit 1.2 state
 609  */
 610 static int radeon_emit_state2(drm_radeon_private_t * dev_priv,
 611                               drm_file_t * filp_priv,
 612                               drm_radeon_state_t * state)
 613 {
 614         RING_LOCALS;
 615
 616         if (state->dirty & RADEON_UPLOAD_ZBIAS) {
 617                 BEGIN_RING(3);
 618                 OUT_RING(CP_PACKET0(RADEON_SE_ZBIAS_FACTOR, 1));
 619                 OUT_RING(state->context2.se_zbias_factor);
 620                 OUT_RING(state->context2.se_zbias_constant);
 621                 ADVANCE_RING();
 622         }
 623
 624         return radeon_emit_state(dev_priv, filp_priv, &state->context,
 625                                  state->tex, state->dirty);
 626 }
 627
 628 /* New (1.3) state mechanism.  3 commands (packet, scalar, vector) in
 629  * 1.3 cmdbuffers allow all previous state to be updated as well as
 630  * the tcl scalar and vector areas.
 631  */
 632 static struct {
 633         int start;
 634         int len;
 635         const char *name;
 636 } packet[RADEON_MAX_STATE_PACKETS] = {
 637         {RADEON_PP_MISC, 7, "RADEON_PP_MISC"},
 638         {RADEON_PP_CNTL, 3, "RADEON_PP_CNTL"},
 639         {RADEON_RB3D_COLORPITCH, 1, "RADEON_RB3D_COLORPITCH"},
 640         {RADEON_RE_LINE_PATTERN, 2, "RADEON_RE_LINE_PATTERN"},
 641         {RADEON_SE_LINE_WIDTH, 1, "RADEON_SE_LINE_WIDTH"},
 642         {RADEON_PP_LUM_MATRIX, 1, "RADEON_PP_LUM_MATRIX"},
 643         {RADEON_PP_ROT_MATRIX_0, 2, "RADEON_PP_ROT_MATRIX_0"},
 644         {RADEON_RB3D_STENCILREFMASK, 3, "RADEON_RB3D_STENCILREFMASK"},
 645         {RADEON_SE_VPORT_XSCALE, 6, "RADEON_SE_VPORT_XSCALE"},
 646         {RADEON_SE_CNTL, 2, "RADEON_SE_CNTL"},
 647         {RADEON_SE_CNTL_STATUS, 1, "RADEON_SE_CNTL_STATUS"},
 648         {RADEON_RE_MISC, 1, "RADEON_RE_MISC"},
 649         {RADEON_PP_TXFILTER_0, 6, "RADEON_PP_TXFILTER_0"},
 650         {RADEON_PP_BORDER_COLOR_0, 1, "RADEON_PP_BORDER_COLOR_0"},
 651         {RADEON_PP_TXFILTER_1, 6, "RADEON_PP_TXFILTER_1"},
 652         {RADEON_PP_BORDER_COLOR_1, 1, "RADEON_PP_BORDER_COLOR_1"},
 653         {RADEON_PP_TXFILTER_2, 6, "RADEON_PP_TXFILTER_2"},
 654         {RADEON_PP_BORDER_COLOR_2, 1, "RADEON_PP_BORDER_COLOR_2"},
 655         {RADEON_SE_ZBIAS_FACTOR, 2, "RADEON_SE_ZBIAS_FACTOR"},
 656         {RADEON_SE_TCL_OUTPUT_VTX_FMT, 11, "RADEON_SE_TCL_OUTPUT_VTX_FMT"},
 657         {RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED, 17,
 658                     "RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED"},
 659         {R200_PP_TXCBLEND_0, 4, "R200_PP_TXCBLEND_0"},
 660         {R200_PP_TXCBLEND_1, 4, "R200_PP_TXCBLEND_1"},
 661         {R200_PP_TXCBLEND_2, 4, "R200_PP_TXCBLEND_2"},
 662         {R200_PP_TXCBLEND_3, 4, "R200_PP_TXCBLEND_3"},
 663         {R200_PP_TXCBLEND_4, 4, "R200_PP_TXCBLEND_4"},
 664         {R200_PP_TXCBLEND_5, 4, "R200_PP_TXCBLEND_5"},
 665         {R200_PP_TXCBLEND_6, 4, "R200_PP_TXCBLEND_6"},
 666         {R200_PP_TXCBLEND_7, 4, "R200_PP_TXCBLEND_7"},
 667         {R200_SE_TCL_LIGHT_MODEL_CTL_0, 6, "R200_SE_TCL_LIGHT_MODEL_CTL_0"},
 668         {R200_PP_TFACTOR_0, 6, "R200_PP_TFACTOR_0"},
 669         {R200_SE_VTX_FMT_0, 4, "R200_SE_VTX_FMT_0"},
 670         {R200_SE_VAP_CNTL, 1, "R200_SE_VAP_CNTL"},
 671         {R200_SE_TCL_MATRIX_SEL_0, 5, "R200_SE_TCL_MATRIX_SEL_0"},
 672         {R200_SE_TCL_TEX_PROC_CTL_2, 5, "R200_SE_TCL_TEX_PROC_CTL_2"},
 673         {R200_SE_TCL_UCP_VERT_BLEND_CTL, 1, "R200_SE_TCL_UCP_VERT_BLEND_CTL"},
 674         {R200_PP_TXFILTER_0, 6, "R200_PP_TXFILTER_0"},
 675         {R200_PP_TXFILTER_1, 6, "R200_PP_TXFILTER_1"},
 676         {R200_PP_TXFILTER_2, 6, "R200_PP_TXFILTER_2"},
 677         {R200_PP_TXFILTER_3, 6, "R200_PP_TXFILTER_3"},
 678         {R200_PP_TXFILTER_4, 6, "R200_PP_TXFILTER_4"},
 679         {R200_PP_TXFILTER_5, 6, "R200_PP_TXFILTER_5"},
 680         {R200_PP_TXOFFSET_0, 1, "R200_PP_TXOFFSET_0"},
 681         {R200_PP_TXOFFSET_1, 1, "R200_PP_TXOFFSET_1"},
 682         {R200_PP_TXOFFSET_2, 1, "R200_PP_TXOFFSET_2"},
 683         {R200_PP_TXOFFSET_3, 1, "R200_PP_TXOFFSET_3"},
 684         {R200_PP_TXOFFSET_4, 1, "R200_PP_TXOFFSET_4"},
 685         {R200_PP_TXOFFSET_5, 1, "R200_PP_TXOFFSET_5"},
 686         {R200_SE_VTE_CNTL, 1, "R200_SE_VTE_CNTL"},
 687         {R200_SE_TCL_OUTPUT_VTX_COMP_SEL, 1,
 688          "R200_SE_TCL_OUTPUT_VTX_COMP_SEL"},
 689         {R200_PP_TAM_DEBUG3, 1, "R200_PP_TAM_DEBUG3"},
 690         {R200_PP_CNTL_X, 1, "R200_PP_CNTL_X"},
 691         {R200_RB3D_DEPTHXY_OFFSET, 1, "R200_RB3D_DEPTHXY_OFFSET"},
 692         {R200_RE_AUX_SCISSOR_CNTL, 1, "R200_RE_AUX_SCISSOR_CNTL"},
 693         {R200_RE_SCISSOR_TL_0, 2, "R200_RE_SCISSOR_TL_0"},
 694         {R200_RE_SCISSOR_TL_1, 2, "R200_RE_SCISSOR_TL_1"},
 695         {R200_RE_SCISSOR_TL_2, 2, "R200_RE_SCISSOR_TL_2"},
 696         {R200_SE_VAP_CNTL_STATUS, 1, "R200_SE_VAP_CNTL_STATUS"},
 697         {R200_SE_VTX_STATE_CNTL, 1, "R200_SE_VTX_STATE_CNTL"},
 698         {R200_RE_POINTSIZE, 1, "R200_RE_POINTSIZE"},
 699         {R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0, 4,
 700                     "R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0"},
 701         {R200_PP_CUBIC_FACES_0, 1, "R200_PP_CUBIC_FACES_0"},    /* 61 */
 702         {R200_PP_CUBIC_OFFSET_F1_0, 5, "R200_PP_CUBIC_OFFSET_F1_0"}, /* 62 */
 703         {R200_PP_CUBIC_FACES_1, 1, "R200_PP_CUBIC_FACES_1"},
 704         {R200_PP_CUBIC_OFFSET_F1_1, 5, "R200_PP_CUBIC_OFFSET_F1_1"},
 705         {R200_PP_CUBIC_FACES_2, 1, "R200_PP_CUBIC_FACES_2"},
 706         {R200_PP_CUBIC_OFFSET_F1_2, 5, "R200_PP_CUBIC_OFFSET_F1_2"},
 707         {R200_PP_CUBIC_FACES_3, 1, "R200_PP_CUBIC_FACES_3"},
 708         {R200_PP_CUBIC_OFFSET_F1_3, 5, "R200_PP_CUBIC_OFFSET_F1_3"},
 709         {R200_PP_CUBIC_FACES_4, 1, "R200_PP_CUBIC_FACES_4"},
 710         {R200_PP_CUBIC_OFFSET_F1_4, 5, "R200_PP_CUBIC_OFFSET_F1_4"},
 711         {R200_PP_CUBIC_FACES_5, 1, "R200_PP_CUBIC_FACES_5"},
 712         {R200_PP_CUBIC_OFFSET_F1_5, 5, "R200_PP_CUBIC_OFFSET_F1_5"},
 713         {RADEON_PP_TEX_SIZE_0, 2, "RADEON_PP_TEX_SIZE_0"},
 714         {RADEON_PP_TEX_SIZE_1, 2, "RADEON_PP_TEX_SIZE_1"},
 715         {RADEON_PP_TEX_SIZE_2, 2, "RADEON_PP_TEX_SIZE_2"},
 716         {R200_RB3D_BLENDCOLOR, 3, "R200_RB3D_BLENDCOLOR"},
 717         {R200_SE_TCL_POINT_SPRITE_CNTL, 1, "R200_SE_TCL_POINT_SPRITE_CNTL"},
 718         {RADEON_PP_CUBIC_FACES_0, 1, "RADEON_PP_CUBIC_FACES_0"},
 719         {RADEON_PP_CUBIC_OFFSET_T0_0, 5, "RADEON_PP_CUBIC_OFFSET_T0_0"},
 720         {RADEON_PP_CUBIC_FACES_1, 1, "RADEON_PP_CUBIC_FACES_1"},
 721         {RADEON_PP_CUBIC_OFFSET_T1_0, 5, "RADEON_PP_CUBIC_OFFSET_T1_0"},
 722         {RADEON_PP_CUBIC_FACES_2, 1, "RADEON_PP_CUBIC_FACES_2"},
 723         {RADEON_PP_CUBIC_OFFSET_T2_0, 5, "RADEON_PP_CUBIC_OFFSET_T2_0"},
 724         {R200_PP_TRI_PERF, 2, "R200_PP_TRI_PERF"},
 725         {R200_PP_AFS_0, 32, "R200_PP_AFS_0"},     /* 85 */
 726         {R200_PP_AFS_1, 32, "R200_PP_AFS_1"},
 727         {R200_PP_TFACTOR_0, 8, "R200_ATF_TFACTOR"},
 728         {R200_PP_TXFILTER_0, 8, "R200_PP_TXCTLALL_0"},
 729         {R200_PP_TXFILTER_1, 8, "R200_PP_TXCTLALL_1"},
 730         {R200_PP_TXFILTER_2, 8, "R200_PP_TXCTLALL_2"},
 731         {R200_PP_TXFILTER_3, 8, "R200_PP_TXCTLALL_3"},
 732         {R200_PP_TXFILTER_4, 8, "R200_PP_TXCTLALL_4"},
 733         {R200_PP_TXFILTER_5, 8, "R200_PP_TXCTLALL_5"},
 734         {R200_VAP_PVS_CNTL_1, 2, "R200_VAP_PVS_CNTL"},
 735 };
 736
 737 /* ================================================================
 738  * Performance monitoring functions
 739  */
 740
 741 static void radeon_clear_box(drm_radeon_private_t * dev_priv,
 742                              int x, int y, int w, int h, int r, int g, int b)
 743 {
 744         u32 color;
 745         RING_LOCALS;
 746
 747         x += dev_priv->sarea_priv->boxes[0].x1;
 748         y += dev_priv->sarea_priv->boxes[0].y1;
 749
 750         switch (dev_priv->color_fmt) {
 751         case RADEON_COLOR_FORMAT_RGB565:
 752                 color = (((r & 0xf8) << 8) |
 753                          ((g & 0xfc) << 3) | ((b & 0xf8) >> 3));
 754                 break;
 755         case RADEON_COLOR_FORMAT_ARGB8888:
 756         default:
 757                 color = (((0xff) << 24) | (r << 16) | (g << 8) | b);
 758                 break;
 759         }
 760
 761         BEGIN_RING(4);
 762         RADEON_WAIT_UNTIL_3D_IDLE();
 763         OUT_RING(CP_PACKET0(RADEON_DP_WRITE_MASK, 0));
 764         OUT_RING(0xffffffff);
 765         ADVANCE_RING();
 766
 767         BEGIN_RING(6);
 768
 769         OUT_RING(CP_PACKET3(RADEON_CNTL_PAINT_MULTI, 4));
 770         OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 771                  RADEON_GMC_BRUSH_SOLID_COLOR |
 772                  (dev_priv->color_fmt << 8) |
 773                  RADEON_GMC_SRC_DATATYPE_COLOR |
 774                  RADEON_ROP3_P | RADEON_GMC_CLR_CMP_CNTL_DIS);
 775
 776         if (dev_priv->page_flipping && dev_priv->current_page == 1) {
 777                 OUT_RING(dev_priv->front_pitch_offset);
 778         } else {
 779                 OUT_RING(dev_priv->back_pitch_offset);
 780         }
 781
 782         OUT_RING(color);
 783
 784         OUT_RING((x << 16) | y);
 785         OUT_RING((w << 16) | h);
 786
 787         ADVANCE_RING();
 788 }
 789
 790 static void radeon_cp_performance_boxes(drm_radeon_private_t * dev_priv)
 791 {
 792         /* Collapse various things into a wait flag -- trying to
 793          * guess if userspase slept -- better just to have them tell us.
 794          */
 795         if (dev_priv->stats.last_frame_reads > 1 ||
 796             dev_priv->stats.last_clear_reads > dev_priv->stats.clears) {
 797                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 798         }
 799
 800         if (dev_priv->stats.freelist_loops) {
 801                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 802         }
 803
 804         /* Purple box for page flipping
 805          */
 806         if (dev_priv->stats.boxes & RADEON_BOX_FLIP)
 807                 radeon_clear_box(dev_priv, 4, 4, 8, 8, 255, 0, 255);
 808
 809         /* Red box if we have to wait for idle at any point
 810          */
 811         if (dev_priv->stats.boxes & RADEON_BOX_WAIT_IDLE)
 812                 radeon_clear_box(dev_priv, 16, 4, 8, 8, 255, 0, 0);
 813
 814         /* Blue box: lost context?
 815          */
 816
 817         /* Yellow box for texture swaps
 818          */
 819         if (dev_priv->stats.boxes & RADEON_BOX_TEXTURE_LOAD)
 820                 radeon_clear_box(dev_priv, 40, 4, 8, 8, 255, 255, 0);
 821
 822         /* Green box if hardware never idles (as far as we can tell)
 823          */
 824         if (!(dev_priv->stats.boxes & RADEON_BOX_DMA_IDLE))
 825                 radeon_clear_box(dev_priv, 64, 4, 8, 8, 0, 255, 0);
 826
 827         /* Draw bars indicating number of buffers allocated
 828          * (not a great measure, easily confused)
 829          */
 830         if (dev_priv->stats.requested_bufs) {
 831                 if (dev_priv->stats.requested_bufs > 100)
 832                         dev_priv->stats.requested_bufs = 100;
 833
 834                 radeon_clear_box(dev_priv, 4, 16,
 835                                  dev_priv->stats.requested_bufs, 4,
 836                                  196, 128, 128);
 837         }
 838
 839         memset(&dev_priv->stats, 0, sizeof(dev_priv->stats));
 840
 841 }
 842
 843 /* ================================================================
 844  * CP command dispatch functions
 845  */
 846
 847 static void radeon_cp_dispatch_clear(drm_device_t * dev,
 848                                      drm_radeon_clear_t * clear,
 849                                      drm_radeon_clear_rect_t * depth_boxes)
 850 {
 851         drm_radeon_private_t *dev_priv = dev->dev_private;
 852         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 853         drm_radeon_depth_clear_t *depth_clear = &dev_priv->depth_clear;
 854         int nbox = sarea_priv->nbox;
 855         drm_clip_rect_t *pbox = sarea_priv->boxes;
 856         unsigned int flags = clear->flags;
 857         u32 rb3d_cntl = 0, rb3d_stencilrefmask = 0;
 858         int i;
 859         RING_LOCALS;
 860         DRM_DEBUG("flags = 0x%x\n", flags);
 861
 862         dev_priv->stats.clears++;
 863
 864         if (dev_priv->page_flipping && dev_priv->current_page == 1) {
 865                 unsigned int tmp = flags;
 866
 867                 flags &= ~(RADEON_FRONT | RADEON_BACK);
 868                 if (tmp & RADEON_FRONT)
 869                         flags |= RADEON_BACK;
 870                 if (tmp & RADEON_BACK)
 871                         flags |= RADEON_FRONT;
 872         }
 873
 874         if (flags & (RADEON_FRONT | RADEON_BACK)) {
 875
 876                 BEGIN_RING(4);
 877
 878                 /* Ensure the 3D stream is idle before doing a
 879                  * 2D fill to clear the front or back buffer.
 880                  */
 881                 RADEON_WAIT_UNTIL_3D_IDLE();
 882
 883                 OUT_RING(CP_PACKET0(RADEON_DP_WRITE_MASK, 0));
 884                 OUT_RING(clear->color_mask);
 885
 886                 ADVANCE_RING();
 887
 888                 /* Make sure we restore the 3D state next time.
 889                  */
 890                 dev_priv->sarea_priv->ctx_owner = 0;
 891
 892                 for (i = 0; i < nbox; i++) {
 893                         int x = pbox[i].x1;
 894                         int y = pbox[i].y1;
 895                         int w = pbox[i].x2 - x;
 896                         int h = pbox[i].y2 - y;
 897
 898                         DRM_DEBUG("dispatch clear %d,%d-%d,%d flags 0x%x\n",
 899                                   x, y, w, h, flags);
 900
 901                         if (flags & RADEON_FRONT) {
 902                                 BEGIN_RING(6);
 903
 904                                 OUT_RING(CP_PACKET3
 905                                          (RADEON_CNTL_PAINT_MULTI, 4));
 906                                 OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 907                                          RADEON_GMC_BRUSH_SOLID_COLOR |
 908                                          (dev_priv->
 909                                           color_fmt << 8) |
 910                                          RADEON_GMC_SRC_DATATYPE_COLOR |
 911                                          RADEON_ROP3_P |
 912                                          RADEON_GMC_CLR_CMP_CNTL_DIS);
 913
 914                                 OUT_RING(dev_priv->front_pitch_offset);
 915                                 OUT_RING(clear->clear_color);
 916
 917                                 OUT_RING((x << 16) | y);
 918                                 OUT_RING((w << 16) | h);
 919
 920                                 ADVANCE_RING();
 921                         }
 922
 923                         if (flags & RADEON_BACK) {
 924                                 BEGIN_RING(6);
 925
 926                                 OUT_RING(CP_PACKET3
 927                                          (RADEON_CNTL_PAINT_MULTI, 4));
 928                                 OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 929                                          RADEON_GMC_BRUSH_SOLID_COLOR |
 930                                          (dev_priv->
 931                                           color_fmt << 8) |
 932                                          RADEON_GMC_SRC_DATATYPE_COLOR |
 933                                          RADEON_ROP3_P |
 934                                          RADEON_GMC_CLR_CMP_CNTL_DIS);
 935
 936                                 OUT_RING(dev_priv->back_pitch_offset);
 937                                 OUT_RING(clear->clear_color);
 938
 939                                 OUT_RING((x << 16) | y);
 940                                 OUT_RING((w << 16) | h);
 941
 942                                 ADVANCE_RING();
 943                         }
 944                 }
 945         }
 946
 947         /* hyper z clear */
 948         /* no docs available, based on reverse engeneering by Stephane Marchesin */
 949         if ((flags & (RADEON_DEPTH | RADEON_STENCIL))
 950             && (flags & RADEON_CLEAR_FASTZ)) {
 951
 952                 int i;
 953                 int depthpixperline =
 954                     dev_priv->depth_fmt ==
 955                     RADEON_DEPTH_FORMAT_16BIT_INT_Z ? (dev_priv->depth_pitch /
 956                                                        2) : (dev_priv->
 957                                                              depth_pitch / 4);
 958
 959                 u32 clearmask;
 960
 961                 u32 tempRB3D_DEPTHCLEARVALUE = clear->clear_depth |
 962                     ((clear->depth_mask & 0xff) << 24);
 963
 964                 /* Make sure we restore the 3D state next time.
 965                  * we haven't touched any "normal" state - still need this?
 966                  */
 967                 dev_priv->sarea_priv->ctx_owner = 0;
 968
 969                 if ((dev_priv->flags & RADEON_HAS_HIERZ)
 970                     && (flags & RADEON_USE_HIERZ)) {
 971                         /* FIXME : reverse engineer that for Rx00 cards */
 972                         /* FIXME : the mask supposedly contains low-res z values. So can't set
 973                            just to the max (0xff? or actually 0x3fff?), need to take z clear
 974                            value into account? */
 975                         /* pattern seems to work for r100, though get slight
 976                            rendering errors with glxgears. If hierz is not enabled for r100,
 977                            only 4 bits which indicate clear (15,16,31,32, all zero) matter, the
 978                            other ones are ignored, and the same clear mask can be used. That's
 979                            very different behaviour than R200 which needs different clear mask
 980                            and different number of tiles to clear if hierz is enabled or not !?!
 981                          */
 982                         clearmask = (0xff << 22) | (0xff << 6) | 0x003f003f;
 983                 } else {
 984                         /* clear mask : chooses the clearing pattern.
 985                            rv250: could be used to clear only parts of macrotiles
 986                            (but that would get really complicated...)?
 987                            bit 0 and 1 (either or both of them ?!?!) are used to
 988                            not clear tile (or maybe one of the bits indicates if the tile is
 989                            compressed or not), bit 2 and 3 to not clear tile 1,...,.
 990                            Pattern is as follows:
 991                            | 0,1 | 4,5 | 8,9 |12,13|16,17|20,21|24,25|28,29|
 992                            bits -------------------------------------------------
 993                            | 2,3 | 6,7 |10,11|14,15|18,19|22,23|26,27|30,31|
 994                            rv100: clearmask covers 2x8 4x1 tiles, but one clear still
 995                            covers 256 pixels ?!?
 996                          */
 997                         clearmask = 0x0;
 998                 }
 999
1000                 BEGIN_RING(8);
1001                 RADEON_WAIT_UNTIL_2D_IDLE();
1002                 OUT_RING_REG(RADEON_RB3D_DEPTHCLEARVALUE,
1003                              tempRB3D_DEPTHCLEARVALUE);
1004                 /* what offset is this exactly ? */
1005                 OUT_RING_REG(RADEON_RB3D_ZMASKOFFSET, 0);
1006                 /* need ctlstat, otherwise get some strange black flickering */
1007                 OUT_RING_REG(RADEON_RB3D_ZCACHE_CTLSTAT,
1008                              RADEON_RB3D_ZC_FLUSH_ALL);
1009                 ADVANCE_RING();
1010
1011                 for (i = 0; i < nbox; i++) {
1012                         int tileoffset, nrtilesx, nrtilesy, j;
1013                         /* it looks like r200 needs rv-style clears, at least if hierz is not enabled? */
1014                         if ((dev_priv->flags & RADEON_HAS_HIERZ)
1015                             && !(dev_priv->microcode_version == UCODE_R200)) {
1016                                 /* FIXME : figure this out for r200 (when hierz is enabled). Or
1017                                    maybe r200 actually doesn't need to put the low-res z value into
1018                                    the tile cache like r100, but just needs to clear the hi-level z-buffer?
1019                                    Works for R100, both with hierz and without.
1020                                    R100 seems to operate on 2x1 8x8 tiles, but...
1021                                    odd: offset/nrtiles need to be 64 pix (4 block) aligned? Potentially
1022                                    problematic with resolutions which are not 64 pix aligned? */
1023                                 tileoffset =
1024                                     ((pbox[i].y1 >> 3) * depthpixperline +
1025                                      pbox[i].x1) >> 6;
1026                                 nrtilesx =
1027                                     ((pbox[i].x2 & ~63) -
1028                                      (pbox[i].x1 & ~63)) >> 4;
1029                                 nrtilesy =
1030                                     (pbox[i].y2 >> 3) - (pbox[i].y1 >> 3);
1031                                 for (j = 0; j <= nrtilesy; j++) {
1032                                         BEGIN_RING(4);
1033                                         OUT_RING(CP_PACKET3
1034                                                  (RADEON_3D_CLEAR_ZMASK, 2));
1035                                         /* first tile */
1036                                         OUT_RING(tileoffset * 8);
1037                                         /* the number of tiles to clear */
1038                                         OUT_RING(nrtilesx + 4);
1039                                         /* clear mask : chooses the clearing pattern. */
1040                                         OUT_RING(clearmask);
1041                                         ADVANCE_RING();
1042                                         tileoffset += depthpixperline >> 6;
1043                                 }
1044                         } else if (dev_priv->microcode_version == UCODE_R200) {
1045                                 /* works for rv250. */
1046                                 /* find first macro tile (8x2 4x4 z-pixels on rv250) */
1047                                 tileoffset =
1048                                     ((pbox[i].y1 >> 3) * depthpixperline +
1049                                      pbox[i].x1) >> 5;
1050                                 nrtilesx =
1051                                     (pbox[i].x2 >> 5) - (pbox[i].x1 >> 5);
1052                                 nrtilesy =
1053                                     (pbox[i].y2 >> 3) - (pbox[i].y1 >> 3);
1054                                 for (j = 0; j <= nrtilesy; j++) {
1055                                         BEGIN_RING(4);
1056                                         OUT_RING(CP_PACKET3
1057                                                  (RADEON_3D_CLEAR_ZMASK, 2));
1058                                         /* first tile */
1059                                         /* judging by the first tile offset needed, could possibly
1060                                            directly address/clear 4x4 tiles instead of 8x2 * 4x4
1061                                            macro tiles, though would still need clear mask for
1062                                            right/bottom if truely 4x4 granularity is desired ? */
1063                                         OUT_RING(tileoffset * 16);
1064                                         /* the number of tiles to clear */
1065                                         OUT_RING(nrtilesx + 1);
1066                                         /* clear mask : chooses the clearing pattern. */
1067                                         OUT_RING(clearmask);
1068                                         ADVANCE_RING();
1069                                         tileoffset += depthpixperline >> 5;
1070                                 }
1071                         } else {        /* rv 100 */
1072                                 /* rv100 might not need 64 pix alignment, who knows */
1073                                 /* offsets are, hmm, weird */
1074                                 tileoffset =
1075                                     ((pbox[i].y1 >> 4) * depthpixperline +
1076                                      pbox[i].x1) >> 6;
1077                                 nrtilesx =
1078                                     ((pbox[i].x2 & ~63) -
1079                                      (pbox[i].x1 & ~63)) >> 4;
1080                                 nrtilesy =
1081                                     (pbox[i].y2 >> 4) - (pbox[i].y1 >> 4);
1082                                 for (j = 0; j <= nrtilesy; j++) {
1083                                         BEGIN_RING(4);
1084                                         OUT_RING(CP_PACKET3
1085                                                  (RADEON_3D_CLEAR_ZMASK, 2));
1086                                         OUT_RING(tileoffset * 128);
1087                                         /* the number of tiles to clear */
1088                                         OUT_RING(nrtilesx + 4);
1089                                         /* clear mask : chooses the clearing pattern. */
1090                                         OUT_RING(clearmask);
1091                                         ADVANCE_RING();
1092                                         tileoffset += depthpixperline >> 6;
1093                                 }
1094                         }
1095                 }
1096
1097                 /* TODO don't always clear all hi-level z tiles */
1098                 if ((dev_priv->flags & RADEON_HAS_HIERZ)
1099                     && (dev_priv->microcode_version == UCODE_R200)
1100                     && (flags & RADEON_USE_HIERZ))
1101                         /* r100 and cards without hierarchical z-buffer have no high-level z-buffer */
1102                         /* FIXME : the mask supposedly contains low-res z values. So can't set
1103                            just to the max (0xff? or actually 0x3fff?), need to take z clear
1104                            value into account? */
1105                 {
1106                         BEGIN_RING(4);
1107                         OUT_RING(CP_PACKET3(RADEON_3D_CLEAR_HIZ, 2));
1108                         OUT_RING(0x0);  /* First tile */
1109                         OUT_RING(0x3cc0);
1110                         OUT_RING((0xff << 22) | (0xff << 6) | 0x003f003f);
1111                         ADVANCE_RING();
1112                 }
1113         }
1114
1115         /* We have to clear the depth and/or stencil buffers by
1116          * rendering a quad into just those buffers.  Thus, we have to
1117          * make sure the 3D engine is configured correctly.
1118          */
1119         else if ((dev_priv->microcode_version == UCODE_R200) &&
1120                 (flags & (RADEON_DEPTH | RADEON_STENCIL))) {
1121
1122                 int tempPP_CNTL;
1123                 int tempRE_CNTL;
1124                 int tempRB3D_CNTL;
1125                 int tempRB3D_ZSTENCILCNTL;
1126                 int tempRB3D_STENCILREFMASK;
1127                 int tempRB3D_PLANEMASK;
1128                 int tempSE_CNTL;
1129                 int tempSE_VTE_CNTL;
1130                 int tempSE_VTX_FMT_0;
1131                 int tempSE_VTX_FMT_1;
1132                 int tempSE_VAP_CNTL;
1133                 int tempRE_AUX_SCISSOR_CNTL;
1134
1135                 tempPP_CNTL = 0;
1136                 tempRE_CNTL = 0;
1137
1138                 tempRB3D_CNTL = depth_clear->rb3d_cntl;
1139
1140                 tempRB3D_ZSTENCILCNTL = depth_clear->rb3d_zstencilcntl;
1141                 tempRB3D_STENCILREFMASK = 0x0;
1142
1143                 tempSE_CNTL = depth_clear->se_cntl;
1144
1145                 /* Disable TCL */
1146
1147                 tempSE_VAP_CNTL = (     /* SE_VAP_CNTL__FORCE_W_TO_ONE_MASK |  */
1148                                           (0x9 <<
1149                                            SE_VAP_CNTL__VF_MAX_VTX_NUM__SHIFT));
1150
1151                 tempRB3D_PLANEMASK = 0x0;
1152
1153                 tempRE_AUX_SCISSOR_CNTL = 0x0;
1154
1155                 tempSE_VTE_CNTL =
1156                     SE_VTE_CNTL__VTX_XY_FMT_MASK | SE_VTE_CNTL__VTX_Z_FMT_MASK;
1157
1158                 /* Vertex format (X, Y, Z, W) */
1159                 tempSE_VTX_FMT_0 =
1160                     SE_VTX_FMT_0__VTX_Z0_PRESENT_MASK |
1161                     SE_VTX_FMT_0__VTX_W0_PRESENT_MASK;
1162                 tempSE_VTX_FMT_1 = 0x0;
1163
1164                 /*
1165                  * Depth buffer specific enables
1166                  */
1167                 if (flags & RADEON_DEPTH) {
1168                         /* Enable depth buffer */
1169                         tempRB3D_CNTL |= RADEON_Z_ENABLE;
1170                 } else {
1171                         /* Disable depth buffer */
1172                         tempRB3D_CNTL &= ~RADEON_Z_ENABLE;
1173                 }
1174
1175                 /*
1176                  * Stencil buffer specific enables
1177                  */
1178                 if (flags & RADEON_STENCIL) {
1179                         tempRB3D_CNTL |= RADEON_STENCIL_ENABLE;
1180                         tempRB3D_STENCILREFMASK = clear->depth_mask;
1181                 } else {
1182                         tempRB3D_CNTL &= ~RADEON_STENCIL_ENABLE;
1183                         tempRB3D_STENCILREFMASK = 0x00000000;
1184                 }
1185
1186                 if (flags & RADEON_USE_COMP_ZBUF) {
1187                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_COMPRESSION_ENABLE |
1188                             RADEON_Z_DECOMPRESSION_ENABLE;
1189                 }
1190                 if (flags & RADEON_USE_HIERZ) {
1191                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_HIERARCHY_ENABLE;
1192                 }
1193
1194                 BEGIN_RING(26);
1195                 RADEON_WAIT_UNTIL_2D_IDLE();
1196
1197                 OUT_RING_REG(RADEON_PP_CNTL, tempPP_CNTL);
1198                 OUT_RING_REG(R200_RE_CNTL, tempRE_CNTL);
1199                 OUT_RING_REG(RADEON_RB3D_CNTL, tempRB3D_CNTL);
1200                 OUT_RING_REG(RADEON_RB3D_ZSTENCILCNTL, tempRB3D_ZSTENCILCNTL);
1201                 OUT_RING_REG(RADEON_RB3D_STENCILREFMASK,
1202                              tempRB3D_STENCILREFMASK);
1203                 OUT_RING_REG(RADEON_RB3D_PLANEMASK, tempRB3D_PLANEMASK);
1204                 OUT_RING_REG(RADEON_SE_CNTL, tempSE_CNTL);
1205                 OUT_RING_REG(R200_SE_VTE_CNTL, tempSE_VTE_CNTL);
1206                 OUT_RING_REG(R200_SE_VTX_FMT_0, tempSE_VTX_FMT_0);
1207                 OUT_RING_REG(R200_SE_VTX_FMT_1, tempSE_VTX_FMT_1);
1208                 OUT_RING_REG(R200_SE_VAP_CNTL, tempSE_VAP_CNTL);
1209                 OUT_RING_REG(R200_RE_AUX_SCISSOR_CNTL, tempRE_AUX_SCISSOR_CNTL);
1210                 ADVANCE_RING();
1211
1212                 /* Make sure we restore the 3D state next time.
1213                  */
1214                 dev_priv->sarea_priv->ctx_owner = 0;
1215
1216                 for (i = 0; i < nbox; i++) {
1217
1218                         /* Funny that this should be required --
1219                          *  sets top-left?
1220                          */
1221                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1222
1223                         BEGIN_RING(14);
1224                         OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 12));
1225                         OUT_RING((RADEON_PRIM_TYPE_RECT_LIST |
1226                                   RADEON_PRIM_WALK_RING |
1227                                   (3 << RADEON_NUM_VERTICES_SHIFT)));
1228                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1229                         OUT_RING(depth_boxes[i].ui[CLEAR_Y1]);
1230                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1231                         OUT_RING(0x3f800000);
1232                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1233                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1234                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1235                         OUT_RING(0x3f800000);
1236                         OUT_RING(depth_boxes[i].ui[CLEAR_X2]);
1237                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1238                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1239                         OUT_RING(0x3f800000);
1240                         ADVANCE_RING();
1241                 }
1242         } else if ((flags & (RADEON_DEPTH | RADEON_STENCIL))) {
1243
1244                 int tempRB3D_ZSTENCILCNTL = depth_clear->rb3d_zstencilcntl;
1245
1246                 rb3d_cntl = depth_clear->rb3d_cntl;
1247
1248                 if (flags & RADEON_DEPTH) {
1249                         rb3d_cntl |= RADEON_Z_ENABLE;
1250                 } else {
1251                         rb3d_cntl &= ~RADEON_Z_ENABLE;
1252                 }
1253
1254                 if (flags & RADEON_STENCIL) {
1255                         rb3d_cntl |= RADEON_STENCIL_ENABLE;
1256                         rb3d_stencilrefmask = clear->depth_mask;        /* misnamed field */
1257                 } else {
1258                         rb3d_cntl &= ~RADEON_STENCIL_ENABLE;
1259                         rb3d_stencilrefmask = 0x00000000;
1260                 }
1261
1262                 if (flags & RADEON_USE_COMP_ZBUF) {
1263                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_COMPRESSION_ENABLE |
1264                             RADEON_Z_DECOMPRESSION_ENABLE;
1265                 }
1266                 if (flags & RADEON_USE_HIERZ) {
1267                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_HIERARCHY_ENABLE;
1268                 }
1269
1270                 BEGIN_RING(13);
1271                 RADEON_WAIT_UNTIL_2D_IDLE();
1272
1273                 OUT_RING(CP_PACKET0(RADEON_PP_CNTL, 1));
1274                 OUT_RING(0x00000000);
1275                 OUT_RING(rb3d_cntl);
1276
1277                 OUT_RING_REG(RADEON_RB3D_ZSTENCILCNTL, tempRB3D_ZSTENCILCNTL);
1278                 OUT_RING_REG(RADEON_RB3D_STENCILREFMASK, rb3d_stencilrefmask);
1279                 OUT_RING_REG(RADEON_RB3D_PLANEMASK, 0x00000000);
1280                 OUT_RING_REG(RADEON_SE_CNTL, depth_clear->se_cntl);
1281                 ADVANCE_RING();
1282
1283                 /* Make sure we restore the 3D state next time.
1284                  */
1285                 dev_priv->sarea_priv->ctx_owner = 0;
1286
1287                 for (i = 0; i < nbox; i++) {
1288
1289                         /* Funny that this should be required --
1290                          *  sets top-left?
1291                          */
1292                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1293
1294                         BEGIN_RING(15);
1295
1296                         OUT_RING(CP_PACKET3(RADEON_3D_DRAW_IMMD, 13));
1297                         OUT_RING(RADEON_VTX_Z_PRESENT |
1298                                  RADEON_VTX_PKCOLOR_PRESENT);
1299                         OUT_RING((RADEON_PRIM_TYPE_RECT_LIST |
1300                                   RADEON_PRIM_WALK_RING |
1301                                   RADEON_MAOS_ENABLE |
1302                                   RADEON_VTX_FMT_RADEON_MODE |
1303                                   (3 << RADEON_NUM_VERTICES_SHIFT)));
1304
1305                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1306                         OUT_RING(depth_boxes[i].ui[CLEAR_Y1]);
1307                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1308                         OUT_RING(0x0);
1309
1310                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1311                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1312                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1313                         OUT_RING(0x0);
1314
1315                         OUT_RING(depth_boxes[i].ui[CLEAR_X2]);
1316                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1317                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1318                         OUT_RING(0x0);
1319
1320                         ADVANCE_RING();
1321                 }
1322         }
1323
1324         /* Increment the clear counter.  The client-side 3D driver must
1325          * wait on this value before performing the clear ioctl.  We
1326          * need this because the card's so damned fast...
1327          */
1328         dev_priv->sarea_priv->last_clear++;
1329
1330         BEGIN_RING(4);
1331
1332         RADEON_CLEAR_AGE(dev_priv->sarea_priv->last_clear);
1333         RADEON_WAIT_UNTIL_IDLE();
1334
1335         ADVANCE_RING();
1336 }
1337
1338 static void radeon_cp_dispatch_swap(drm_device_t * dev)
1339 {
1340         drm_radeon_private_t *dev_priv = dev->dev_private;
1341         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1342         int nbox = sarea_priv->nbox;
1343         drm_clip_rect_t *pbox = sarea_priv->boxes;
1344         int i;
1345         RING_LOCALS;
1346         DRM_DEBUG("\n");
1347
1348         /* Do some trivial performance monitoring...
1349          */
1350         if (dev_priv->do_boxes)
1351                 radeon_cp_performance_boxes(dev_priv);
1352
1353         /* Wait for the 3D stream to idle before dispatching the bitblt.
1354          * This will prevent data corruption between the two streams.
1355          */
1356         BEGIN_RING(2);
1357
1358         RADEON_WAIT_UNTIL_3D_IDLE();
1359
1360         ADVANCE_RING();
1361
1362         for (i = 0; i < nbox; i++) {
1363                 int x = pbox[i].x1;
1364                 int y = pbox[i].y1;
1365                 int w = pbox[i].x2 - x;
1366                 int h = pbox[i].y2 - y;
1367
1368                 DRM_DEBUG("dispatch swap %d,%d-%d,%d\n", x, y, w, h);
1369
1370                 BEGIN_RING(9);
1371
1372                 OUT_RING(CP_PACKET0(RADEON_DP_GUI_MASTER_CNTL, 0));
1373                 OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
1374                          RADEON_GMC_DST_PITCH_OFFSET_CNTL |
1375                          RADEON_GMC_BRUSH_NONE |
1376                          (dev_priv->color_fmt << 8) |
1377                          RADEON_GMC_SRC_DATATYPE_COLOR |
1378                          RADEON_ROP3_S |
1379                          RADEON_DP_SRC_SOURCE_MEMORY |
1380                          RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
1381
1382                 /* Make this work even if front & back are flipped:
1383                  */
1384                 OUT_RING(CP_PACKET0(RADEON_SRC_PITCH_OFFSET, 1));
1385                 if (dev_priv->current_page == 0) {
1386                         OUT_RING(dev_priv->back_pitch_offset);
1387                         OUT_RING(dev_priv->front_pitch_offset);
1388                 } else {
1389                         OUT_RING(dev_priv->front_pitch_offset);
1390                         OUT_RING(dev_priv->back_pitch_offset);
1391                 }
1392
1393                 OUT_RING(CP_PACKET0(RADEON_SRC_X_Y, 2));
1394                 OUT_RING((x << 16) | y);
1395                 OUT_RING((x << 16) | y);
1396                 OUT_RING((w << 16) | h);
1397
1398                 ADVANCE_RING();
1399         }
1400
1401         /* Increment the frame counter.  The client-side 3D driver must
1402          * throttle the framerate by waiting for this value before
1403          * performing the swapbuffer ioctl.
1404          */
1405         dev_priv->sarea_priv->last_frame++;
1406
1407         BEGIN_RING(4);
1408
1409         RADEON_FRAME_AGE(dev_priv->sarea_priv->last_frame);
1410         RADEON_WAIT_UNTIL_2D_IDLE();
1411
1412         ADVANCE_RING();
1413 }
1414
1415 static void radeon_cp_dispatch_flip(drm_device_t * dev)
1416 {
1417         drm_radeon_private_t *dev_priv = dev->dev_private;
1418         drm_sarea_t *sarea = (drm_sarea_t *) dev_priv->sarea->handle;
1419         int offset = (dev_priv->current_page == 1)
1420             ? dev_priv->front_offset : dev_priv->back_offset;
1421         RING_LOCALS;
1422         DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
1423                   __FUNCTION__,
1424                   dev_priv->current_page, dev_priv->sarea_priv->pfCurrentPage);
1425
1426         /* Do some trivial performance monitoring...
1427          */
1428         if (dev_priv->do_boxes) {
1429                 dev_priv->stats.boxes |= RADEON_BOX_FLIP;
1430                 radeon_cp_performance_boxes(dev_priv);
1431         }
1432
1433         /* Update the frame offsets for both CRTCs
1434          */
1435         BEGIN_RING(6);
1436
1437         RADEON_WAIT_UNTIL_3D_IDLE();
1438         OUT_RING_REG(RADEON_CRTC_OFFSET,
1439                      ((sarea->frame.y * dev_priv->front_pitch +
1440                        sarea->frame.x * (dev_priv->color_fmt - 2)) & ~7)
1441                      + offset);
1442         OUT_RING_REG(RADEON_CRTC2_OFFSET, dev_priv->sarea_priv->crtc2_base
1443                      + offset);
1444
1445         ADVANCE_RING();
1446
1447         /* Increment the frame counter.  The client-side 3D driver must
1448          * throttle the framerate by waiting for this value before
1449          * performing the swapbuffer ioctl.
1450          */
1451         dev_priv->sarea_priv->last_frame++;
1452         dev_priv->sarea_priv->pfCurrentPage = dev_priv->current_page =
1453             1 - dev_priv->current_page;
1454
1455         BEGIN_RING(2);
1456
1457         RADEON_FRAME_AGE(dev_priv->sarea_priv->last_frame);
1458
1459         ADVANCE_RING();
1460 }
1461
1462 static int bad_prim_vertex_nr(int primitive, int nr)
1463 {
1464         switch (primitive & RADEON_PRIM_TYPE_MASK) {
1465         case RADEON_PRIM_TYPE_NONE:
1466         case RADEON_PRIM_TYPE_POINT:
1467                 return nr < 1;
1468         case RADEON_PRIM_TYPE_LINE:
1469                 return (nr & 1) || nr == 0;
1470         case RADEON_PRIM_TYPE_LINE_STRIP:
1471                 return nr < 2;
1472         case RADEON_PRIM_TYPE_TRI_LIST:
1473         case RADEON_PRIM_TYPE_3VRT_POINT_LIST:
1474         case RADEON_PRIM_TYPE_3VRT_LINE_LIST:
1475         case RADEON_PRIM_TYPE_RECT_LIST:
1476                 return nr % 3 || nr == 0;
1477         case RADEON_PRIM_TYPE_TRI_FAN:
1478         case RADEON_PRIM_TYPE_TRI_STRIP:
1479                 return nr < 3;
1480         default:
1481                 return 1;
1482         }
1483 }
1484
1485 typedef struct {
1486         unsigned int start;
1487         unsigned int finish;
1488         unsigned int prim;
1489         unsigned int numverts;
1490         unsigned int offset;
1491         unsigned int vc_format;
1492 } drm_radeon_tcl_prim_t;
1493
1494 static void radeon_cp_dispatch_vertex(drm_device_t * dev,
1495                                       drm_buf_t * buf,
1496                                       drm_radeon_tcl_prim_t * prim)
1497 {
1498         drm_radeon_private_t *dev_priv = dev->dev_private;
1499         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1500         int offset = dev_priv->gart_buffers_offset + buf->offset + prim->start;
1501         int numverts = (int)prim->numverts;
1502         int nbox = sarea_priv->nbox;
1503         int i = 0;
1504         RING_LOCALS;
1505
1506         DRM_DEBUG("hwprim 0x%x vfmt 0x%x %d..%d %d verts\n",
1507                   prim->prim,
1508                   prim->vc_format, prim->start, prim->finish, prim->numverts);
1509
1510         if (bad_prim_vertex_nr(prim->prim, prim->numverts)) {
1511                 DRM_ERROR("bad prim %x numverts %d\n",
1512                           prim->prim, prim->numverts);
1513                 return;
1514         }
1515
1516         do {
1517                 /* Emit the next cliprect */
1518                 if (i < nbox) {
1519                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1520                 }
1521
1522                 /* Emit the vertex buffer rendering commands */
1523                 BEGIN_RING(5);
1524
1525                 OUT_RING(CP_PACKET3(RADEON_3D_RNDR_GEN_INDX_PRIM, 3));
1526                 OUT_RING(offset);
1527                 OUT_RING(numverts);
1528                 OUT_RING(prim->vc_format);
1529                 OUT_RING(prim->prim | RADEON_PRIM_WALK_LIST |
1530                          RADEON_COLOR_ORDER_RGBA |
1531                          RADEON_VTX_FMT_RADEON_MODE |
1532                          (numverts << RADEON_NUM_VERTICES_SHIFT));
1533
1534                 ADVANCE_RING();
1535
1536                 i++;
1537         } while (i < nbox);
1538 }
1539
1540 static void radeon_cp_discard_buffer(drm_device_t * dev, drm_buf_t * buf)
1541 {
1542         drm_radeon_private_t *dev_priv = dev->dev_private;
1543         drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1544         RING_LOCALS;
1545
1546         buf_priv->age = ++dev_priv->sarea_priv->last_dispatch;
1547
1548         /* Emit the vertex buffer age */
1549         BEGIN_RING(2);
1550         RADEON_DISPATCH_AGE(buf_priv->age);
1551         ADVANCE_RING();
1552
1553         buf->pending = 1;
1554         buf->used = 0;
1555 }
1556
1557 static void radeon_cp_dispatch_indirect(drm_device_t * dev,
1558                                         drm_buf_t * buf, int start, int end)
1559 {
1560         drm_radeon_private_t *dev_priv = dev->dev_private;
1561         RING_LOCALS;
1562         DRM_DEBUG("indirect: buf=%d s=0x%x e=0x%x\n", buf->idx, start, end);
1563
1564         if (start != end) {
1565                 int offset = (dev_priv->gart_buffers_offset
1566                               + buf->offset + start);
1567                 int dwords = (end - start + 3) / sizeof(u32);
1568
1569                 /* Indirect buffer data must be an even number of
1570                  * dwords, so if we've been given an odd number we must
1571                  * pad the data with a Type-2 CP packet.
1572                  */
1573                 if (dwords & 1) {
1574                         u32 *data = (u32 *)
1575                             ((char *)dev->agp_buffer_map->handle
1576                              + buf->offset + start);
1577                         data[dwords++] = RADEON_CP_PACKET2;
1578                 }
1579
1580                 /* Fire off the indirect buffer */
1581                 BEGIN_RING(3);
1582
1583                 OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
1584                 OUT_RING(offset);
1585                 OUT_RING(dwords);
1586
1587                 ADVANCE_RING();
1588         }
1589 }
1590
1591 static void radeon_cp_dispatch_indices(drm_device_t * dev,
1592                                        drm_buf_t * elt_buf,
1593                                        drm_radeon_tcl_prim_t * prim)
1594 {
1595         drm_radeon_private_t *dev_priv = dev->dev_private;
1596         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1597         int offset = dev_priv->gart_buffers_offset + prim->offset;
1598         u32 *data;
1599         int dwords;
1600         int i = 0;
1601         int start = prim->start + RADEON_INDEX_PRIM_OFFSET;
1602         int count = (prim->finish - start) / sizeof(u16);
1603         int nbox = sarea_priv->nbox;
1604
1605         DRM_DEBUG("hwprim 0x%x vfmt 0x%x %d..%d offset: %x nr %d\n",
1606                   prim->prim,
1607                   prim->vc_format,
1608                   prim->start, prim->finish, prim->offset, prim->numverts);
1609
1610         if (bad_prim_vertex_nr(prim->prim, count)) {
1611                 DRM_ERROR("bad prim %x count %d\n", prim->prim, count);
1612                 return;
1613         }
1614
1615         if (start >= prim->finish || (prim->start & 0x7)) {
1616                 DRM_ERROR("buffer prim %d\n", prim->prim);
1617                 return;
1618         }
1619
1620         dwords = (prim->finish - prim->start + 3) / sizeof(u32);
1621
1622         data = (u32 *) ((char *)dev->agp_buffer_map->handle +
1623                         elt_buf->offset + prim->start);
1624
1625         data[0] = CP_PACKET3(RADEON_3D_RNDR_GEN_INDX_PRIM, dwords - 2);
1626         data[1] = offset;
1627         data[2] = prim->numverts;
1628         data[3] = prim->vc_format;
1629         data[4] = (prim->prim |
1630                    RADEON_PRIM_WALK_IND |
1631                    RADEON_COLOR_ORDER_RGBA |
1632                    RADEON_VTX_FMT_RADEON_MODE |
1633                    (count << RADEON_NUM_VERTICES_SHIFT));
1634
1635         do {
1636                 if (i < nbox)
1637                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1638
1639                 radeon_cp_dispatch_indirect(dev, elt_buf,
1640                                             prim->start, prim->finish);
1641
1642                 i++;
1643         } while (i < nbox);
1644
1645 }
1646
1647 #define RADEON_MAX_TEXTURE_SIZE RADEON_BUFFER_SIZE
1648
1649 static int radeon_cp_dispatch_texture(DRMFILE filp,
1650                                       drm_device_t * dev,
1651                                       drm_radeon_texture_t * tex,
1652                                       drm_radeon_tex_image_t * image)
1653 {
1654         drm_radeon_private_t *dev_priv = dev->dev_private;
1655         drm_file_t *filp_priv;
1656         drm_buf_t *buf;
1657         u32 format;
1658         u32 *buffer;
1659         const u8 __user *data;
1660         int size, dwords, tex_width, blit_width, spitch;
1661         u32 height;
1662         int i;
1663         u32 texpitch, microtile;
1664         u32 offset;
1665         RING_LOCALS;
1666
1667         DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
1668
1669         if (radeon_check_and_fixup_offset(dev_priv, filp_priv, &tex->offset)) {
1670                 DRM_ERROR("Invalid destination offset\n");
1671                 return DRM_ERR(EINVAL);
1672         }
1673
1674         dev_priv->stats.boxes |= RADEON_BOX_TEXTURE_LOAD;
1675
1676         /* Flush the pixel cache.  This ensures no pixel data gets mixed
1677          * up with the texture data from the host data blit, otherwise
1678          * part of the texture image may be corrupted.
1679          */
1680         BEGIN_RING(4);
1681         RADEON_FLUSH_CACHE();
1682         RADEON_WAIT_UNTIL_IDLE();
1683         ADVANCE_RING();
1684
1685         /* The compiler won't optimize away a division by a variable,
1686          * even if the only legal values are powers of two.  Thus, we'll
1687          * use a shift instead.
1688          */
1689         switch (tex->format) {
1690         case RADEON_TXFORMAT_ARGB8888:
1691         case RADEON_TXFORMAT_RGBA8888:
1692                 format = RADEON_COLOR_FORMAT_ARGB8888;
1693                 tex_width = tex->width * 4;
1694                 blit_width = image->width * 4;
1695                 break;
1696         case RADEON_TXFORMAT_AI88:
1697         case RADEON_TXFORMAT_ARGB1555:
1698         case RADEON_TXFORMAT_RGB565:
1699         case RADEON_TXFORMAT_ARGB4444:
1700         case RADEON_TXFORMAT_VYUY422:
1701         case RADEON_TXFORMAT_YVYU422:
1702                 format = RADEON_COLOR_FORMAT_RGB565;
1703                 tex_width = tex->width * 2;
1704                 blit_width = image->width * 2;
1705                 break;
1706         case RADEON_TXFORMAT_I8:
1707         case RADEON_TXFORMAT_RGB332:
1708                 format = RADEON_COLOR_FORMAT_CI8;
1709                 tex_width = tex->width * 1;
1710                 blit_width = image->width * 1;
1711                 break;
1712         default:
1713                 DRM_ERROR("invalid texture format %d\n", tex->format);
1714                 return DRM_ERR(EINVAL);
1715         }
1716         spitch = blit_width >> 6;
1717         if (spitch == 0 && image->height > 1)
1718                 return DRM_ERR(EINVAL);
1719
1720         texpitch = tex->pitch;
1721         if ((texpitch << 22) & RADEON_DST_TILE_MICRO) {
1722                 microtile = 1;
1723                 if (tex_width < 64) {
1724                         texpitch &= ~(RADEON_DST_TILE_MICRO >> 22);
1725                         /* we got tiled coordinates, untile them */
1726                         image->x *= 2;
1727                 }
1728         } else
1729                 microtile = 0;
1730
1731         DRM_DEBUG("tex=%dx%d blit=%d\n", tex_width, tex->height, blit_width);
1732
1733         do {
1734                 DRM_DEBUG("tex: ofs=0x%x p=%d f=%d x=%hd y=%hd w=%hd h=%hd\n",
1735                           tex->offset >> 10, tex->pitch, tex->format,
1736                           image->x, image->y, image->width, image->height);
1737
1738                 /* Make a copy of some parameters in case we have to
1739                  * update them for a multi-pass texture blit.
1740                  */
1741                 height = image->height;
1742                 data = (const u8 __user *)image->data;
1743
1744                 size = height * blit_width;
1745
1746                 if (size > RADEON_MAX_TEXTURE_SIZE) {
1747                         height = RADEON_MAX_TEXTURE_SIZE / blit_width;
1748                         size = height * blit_width;
1749                 } else if (size < 4 && size > 0) {
1750                         size = 4;
1751                 } else if (size == 0) {
1752                         return 0;
1753                 }
1754
1755                 buf = radeon_freelist_get(dev);
1756                 if (0 && !buf) {
1757                         radeon_do_cp_idle(dev_priv);
1758                         buf = radeon_freelist_get(dev);
1759                 }
1760                 if (!buf) {
1761                         DRM_DEBUG("radeon_cp_dispatch_texture: EAGAIN\n");
1762                         if (DRM_COPY_TO_USER(tex->image, image, sizeof(*image)))
1763                                 return DRM_ERR(EFAULT);
1764                         return DRM_ERR(EAGAIN);
1765                 }
1766
1767                 /* Dispatch the indirect buffer.
1768                  */
1769                 buffer =
1770                     (u32 *) ((char *)dev->agp_buffer_map->handle + buf->offset);
1771                 dwords = size / 4;
1772
1773 #define RADEON_COPY_MT(_buf, _data, _width) \
1774         do { \
1775                 if (DRM_COPY_FROM_USER(_buf, _data, (_width))) {\
1776                         DRM_ERROR("EFAULT on pad, %d bytes\n", (_width)); \
1777                         return DRM_ERR(EFAULT); \
1778                 } \
1779         } while(0)
1780
1781                 if (microtile) {
1782                         /* texture micro tiling in use, minimum texture width is thus 16 bytes.
1783                            however, we cannot use blitter directly for texture width < 64 bytes,
1784                            since minimum tex pitch is 64 bytes and we need this to match
1785                            the texture width, otherwise the blitter will tile it wrong.
1786                            Thus, tiling manually in this case. Additionally, need to special
1787                            case tex height = 1, since our actual image will have height 2
1788                            and we need to ensure we don't read beyond the texture size
1789                            from user space. */
1790                         if (tex->height == 1) {
1791                                 if (tex_width >= 64 || tex_width <= 16) {
1792                                         RADEON_COPY_MT(buffer, data,
1793                                                 (int)(tex_width * sizeof(u32)));
1794                                 } else if (tex_width == 32) {
1795                                         RADEON_COPY_MT(buffer, data, 16);
1796                                         RADEON_COPY_MT(buffer + 8,
1797                                                        data + 16, 16);
1798                                 }
1799                         } else if (tex_width >= 64 || tex_width == 16) {
1800                                 RADEON_COPY_MT(buffer, data,
1801                                                (int)(dwords * sizeof(u32)));
1802                         } else if (tex_width < 16) {
1803                                 for (i = 0; i < tex->height; i++) {
1804                                         RADEON_COPY_MT(buffer, data, tex_width);
1805                                         buffer += 4;
1806                                         data += tex_width;
1807                                 }
1808                         } else if (tex_width == 32) {
1809                                 /* TODO: make sure this works when not fitting in one buffer
1810                                    (i.e. 32bytes x 2048...) */
1811                                 for (i = 0; i < tex->height; i += 2) {
1812                                         RADEON_COPY_MT(buffer, data, 16);
1813                                         data += 16;
1814                                         RADEON_COPY_MT(buffer + 8, data, 16);
1815                                         data += 16;
1816                                         RADEON_COPY_MT(buffer + 4, data, 16);
1817                                         data += 16;
1818                                         RADEON_COPY_MT(buffer + 12, data, 16);
1819                                         data += 16;
1820                                         buffer += 16;
1821                                 }
1822                         }
1823                 } else {
1824                         if (tex_width >= 32) {
1825                                 /* Texture image width is larger than the minimum, so we
1826                                  * can upload it directly.
1827                                  */
1828                                 RADEON_COPY_MT(buffer, data,
1829                                                (int)(dwords * sizeof(u32)));
1830                         } else {
1831                                 /* Texture image width is less than the minimum, so we
1832                                  * need to pad out each image scanline to the minimum
1833                                  * width.
1834                                  */
1835                                 for (i = 0; i < tex->height; i++) {
1836                                         RADEON_COPY_MT(buffer, data, tex_width);
1837                                         buffer += 8;
1838                                         data += tex_width;
1839                                 }
1840                         }
1841                 }
1842
1843 #undef RADEON_COPY_MT
1844                 buf->filp = filp;
1845                 buf->used = size;
1846                 offset = dev_priv->gart_buffers_offset + buf->offset;
1847                 BEGIN_RING(9);
1848                 OUT_RING(CP_PACKET3(RADEON_CNTL_BITBLT_MULTI, 5));
1849                 OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
1850                          RADEON_GMC_DST_PITCH_OFFSET_CNTL |
1851                          RADEON_GMC_BRUSH_NONE |
1852                          (format << 8) |
1853                          RADEON_GMC_SRC_DATATYPE_COLOR |
1854                          RADEON_ROP3_S |
1855                          RADEON_DP_SRC_SOURCE_MEMORY |
1856                          RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
1857                 OUT_RING((spitch << 22) | (offset >> 10));
1858                 OUT_RING((texpitch << 22) | (tex->offset >> 10));
1859                 OUT_RING(0);
1860                 OUT_RING((image->x << 16) | image->y);
1861                 OUT_RING((image->width << 16) | height);
1862                 RADEON_WAIT_UNTIL_2D_IDLE();
1863                 ADVANCE_RING();
1864
1865                 radeon_cp_discard_buffer(dev, buf);
1866
1867                 /* Update the input parameters for next time */
1868                 image->y += height;
1869                 image->height -= height;
1870                 image->data = (const u8 __user *)image->data + size;
1871         } while (image->height > 0);
1872
1873         /* Flush the pixel cache after the blit completes.  This ensures
1874          * the texture data is written out to memory before rendering
1875          * continues.
1876          */
1877         BEGIN_RING(4);
1878         RADEON_FLUSH_CACHE();
1879         RADEON_WAIT_UNTIL_2D_IDLE();
1880         ADVANCE_RING();
1881         return 0;
1882 }
1883
1884 static void radeon_cp_dispatch_stipple(drm_device_t * dev, u32 * stipple)
1885 {
1886         drm_radeon_private_t *dev_priv = dev->dev_private;
1887         int i;
1888         RING_LOCALS;
1889         DRM_DEBUG("\n");
1890
1891         BEGIN_RING(35);
1892
1893         OUT_RING(CP_PACKET0(RADEON_RE_STIPPLE_ADDR, 0));
1894         OUT_RING(0x00000000);
1895
1896         OUT_RING(CP_PACKET0_TABLE(RADEON_RE_STIPPLE_DATA, 31));
1897         for (i = 0; i < 32; i++) {
1898                 OUT_RING(stipple[i]);
1899         }
1900
1901         ADVANCE_RING();
1902 }
1903
1904 static void radeon_apply_surface_regs(int surf_index,
1905                                       drm_radeon_private_t *dev_priv)
1906 {
1907         if (!dev_priv->mmio)
1908                 return;
1909
1910         radeon_do_cp_idle(dev_priv);
1911
1912         RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * surf_index,
1913                      dev_priv->surfaces[surf_index].flags);
1914         RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16 * surf_index,
1915                      dev_priv->surfaces[surf_index].lower);
1916         RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16 * surf_index,
1917                      dev_priv->surfaces[surf_index].upper);
1918 }
1919
1920 /* Allocates a virtual surface
1921  * doesn't always allocate a real surface, will stretch an existing
1922  * surface when possible.
1923  *
1924  * Note that refcount can be at most 2, since during a free refcount=3
1925  * might mean we have to allocate a new surface which might not always
1926  * be available.
1927  * For example : we allocate three contigous surfaces ABC. If B is
1928  * freed, we suddenly need two surfaces to store A and C, which might
1929  * not always be available.
1930  */
1931 static int alloc_surface(drm_radeon_surface_alloc_t *new,
1932                          drm_radeon_private_t *dev_priv, DRMFILE filp)
1933 {
1934         struct radeon_virt_surface *s;
1935         int i;
1936         int virt_surface_index;
1937         uint32_t new_upper, new_lower;
1938
1939         new_lower = new->address;
1940         new_upper = new_lower + new->size - 1;
1941
1942         /* sanity check */
1943         if ((new_lower >= new_upper) || (new->flags == 0) || (new->size == 0) ||
1944             ((new_upper & RADEON_SURF_ADDRESS_FIXED_MASK) !=
1945              RADEON_SURF_ADDRESS_FIXED_MASK)
1946             || ((new_lower & RADEON_SURF_ADDRESS_FIXED_MASK) != 0))
1947                 return -1;
1948
1949         /* make sure there is no overlap with existing surfaces */
1950         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1951                 if ((dev_priv->surfaces[i].refcount != 0) &&
1952                     (((new_lower >= dev_priv->surfaces[i].lower) &&
1953                       (new_lower < dev_priv->surfaces[i].upper)) ||
1954                      ((new_lower < dev_priv->surfaces[i].lower) &&
1955                       (new_upper > dev_priv->surfaces[i].lower)))) {
1956                         return -1;
1957                 }
1958         }
1959
1960         /* find a virtual surface */
1961         for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++)
1962                 if (dev_priv->virt_surfaces[i].filp == 0)
1963                         break;
1964         if (i == 2 * RADEON_MAX_SURFACES) {
1965                 return -1;
1966         }
1967         virt_surface_index = i;
1968
1969         /* try to reuse an existing surface */
1970         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1971                 /* extend before */
1972                 if ((dev_priv->surfaces[i].refcount == 1) &&
1973                     (new->flags == dev_priv->surfaces[i].flags) &&
1974                     (new_upper + 1 == dev_priv->surfaces[i].lower)) {
1975                         s = &(dev_priv->virt_surfaces[virt_surface_index]);
1976                         s->surface_index = i;
1977                         s->lower = new_lower;
1978                         s->upper = new_upper;
1979                         s->flags = new->flags;
1980                         s->filp = filp;
1981                         dev_priv->surfaces[i].refcount++;
1982                         dev_priv->surfaces[i].lower = s->lower;
1983                         radeon_apply_surface_regs(s->surface_index, dev_priv);
1984                         return virt_surface_index;
1985                 }
1986
1987                 /* extend after */
1988                 if ((dev_priv->surfaces[i].refcount == 1) &&
1989                     (new->flags == dev_priv->surfaces[i].flags) &&
1990                     (new_lower == dev_priv->surfaces[i].upper + 1)) {
1991                         s = &(dev_priv->virt_surfaces[virt_surface_index]);
1992                         s->surface_index = i;
1993                         s->lower = new_lower;
1994                         s->upper = new_upper;
1995                         s->flags = new->flags;
1996                         s->filp = filp;
1997                         dev_priv->surfaces[i].refcount++;
1998                         dev_priv->surfaces[i].upper = s->upper;
1999                         radeon_apply_surface_regs(s->surface_index, dev_priv);
2000                         return virt_surface_index;
2001                 }
2002         }
2003
2004         /* okay, we need a new one */
2005         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
2006                 if (dev_priv->surfaces[i].refcount == 0) {
2007                         s = &(dev_priv->virt_surfaces[virt_surface_index]);
2008                         s->surface_index = i;
2009                         s->lower = new_lower;
2010                         s->upper = new_upper;
2011                         s->flags = new->flags;
2012                         s->filp = filp;
2013                         dev_priv->surfaces[i].refcount = 1;
2014                         dev_priv->surfaces[i].lower = s->lower;
2015                         dev_priv->surfaces[i].upper = s->upper;
2016                         dev_priv->surfaces[i].flags = s->flags;
2017                         radeon_apply_surface_regs(s->surface_index, dev_priv);
2018                         return virt_surface_index;
2019                 }
2020         }
2021
2022         /* we didn't find anything */
2023         return -1;
2024 }
2025
2026 static int free_surface(DRMFILE filp, drm_radeon_private_t * dev_priv,
2027                         int lower)
2028 {
2029         struct radeon_virt_surface *s;
2030         int i;
2031         /* find the virtual surface */
2032         for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
2033                 s = &(dev_priv->virt_surfaces[i]);
2034                 if (s->filp) {
2035                         if ((lower == s->lower) && (filp == s->filp)) {
2036                                 if (dev_priv->surfaces[s->surface_index].
2037                                     lower == s->lower)
2038                                         dev_priv->surfaces[s->surface_index].
2039                                             lower = s->upper;
2040
2041                                 if (dev_priv->surfaces[s->surface_index].
2042                                     upper == s->upper)
2043                                         dev_priv->surfaces[s->surface_index].
2044                                             upper = s->lower;
2045
2046                                 dev_priv->surfaces[s->surface_index].refcount--;
2047                                 if (dev_priv->surfaces[s->surface_index].
2048                                     refcount == 0)
2049                                         dev_priv->surfaces[s->surface_index].
2050                                             flags = 0;
2051                                 s->filp = NULL;
2052                                 radeon_apply_surface_regs(s->surface_index,
2053                                                           dev_priv);
2054                                 return 0;
2055                         }
2056                 }
2057         }
2058         return 1;
2059 }
2060
2061 static void radeon_surfaces_release(DRMFILE filp,
2062                                     drm_radeon_private_t * dev_priv)
2063 {
2064         int i;
2065         for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
2066                 if (dev_priv->virt_surfaces[i].filp == filp)
2067                         free_surface(filp, dev_priv,
2068                                      dev_priv->virt_surfaces[i].lower);
2069         }
2070 }
2071
2072 /* ================================================================
2073  * IOCTL functions
2074  */
2075 static int radeon_surface_alloc(DRM_IOCTL_ARGS)
2076 {
2077         DRM_DEVICE;
2078         drm_radeon_private_t *dev_priv = dev->dev_private;
2079         drm_radeon_surface_alloc_t alloc;
2080
2081         DRM_COPY_FROM_USER_IOCTL(alloc,
2082                                  (drm_radeon_surface_alloc_t __user *) data,
2083                                  sizeof(alloc));
2084
2085         if (alloc_surface(&alloc, dev_priv, filp) == -1)
2086                 return DRM_ERR(EINVAL);
2087         else
2088                 return 0;
2089 }
2090
2091 static int radeon_surface_free(DRM_IOCTL_ARGS)
2092 {
2093         DRM_DEVICE;
2094         drm_radeon_private_t *dev_priv = dev->dev_private;
2095         drm_radeon_surface_free_t memfree;
2096
2097         DRM_COPY_FROM_USER_IOCTL(memfree, (drm_radeon_surface_free_t __user *) data,
2098                                  sizeof(memfree));
2099
2100         if (free_surface(filp, dev_priv, memfree.address))
2101                 return DRM_ERR(EINVAL);
2102         else
2103                 return 0;
2104 }
2105
2106 static int radeon_cp_clear(DRM_IOCTL_ARGS)
2107 {
2108         DRM_DEVICE;
2109         drm_radeon_private_t *dev_priv = dev->dev_private;
2110         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2111         drm_radeon_clear_t clear;
2112         drm_radeon_clear_rect_t depth_boxes[RADEON_NR_SAREA_CLIPRECTS];
2113         DRM_DEBUG("\n");
2114
2115         LOCK_TEST_WITH_RETURN(dev, filp);
2116
2117         DRM_COPY_FROM_USER_IOCTL(clear, (drm_radeon_clear_t __user *) data,
2118                                  sizeof(clear));
2119
2120         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2121
2122         if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
2123                 sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
2124
2125         if (DRM_COPY_FROM_USER(&depth_boxes, clear.depth_boxes,
2126                                sarea_priv->nbox * sizeof(depth_boxes[0])))
2127                 return DRM_ERR(EFAULT);
2128
2129         radeon_cp_dispatch_clear(dev, &clear, depth_boxes);
2130
2131         COMMIT_RING();
2132         return 0;
2133 }
2134
2135 /* Not sure why this isn't set all the time:
2136  */
2137 static int radeon_do_init_pageflip(drm_device_t * dev)
2138 {
2139         drm_radeon_private_t *dev_priv = dev->dev_private;
2140         RING_LOCALS;
2141
2142         DRM_DEBUG("\n");
2143
2144         BEGIN_RING(6);
2145         RADEON_WAIT_UNTIL_3D_IDLE();
2146         OUT_RING(CP_PACKET0(RADEON_CRTC_OFFSET_CNTL, 0));
2147         OUT_RING(RADEON_READ(RADEON_CRTC_OFFSET_CNTL) |
2148                  RADEON_CRTC_OFFSET_FLIP_CNTL);
2149         OUT_RING(CP_PACKET0(RADEON_CRTC2_OFFSET_CNTL, 0));
2150         OUT_RING(RADEON_READ(RADEON_CRTC2_OFFSET_CNTL) |
2151                  RADEON_CRTC_OFFSET_FLIP_CNTL);
2152         ADVANCE_RING();
2153
2154         dev_priv->page_flipping = 1;
2155         dev_priv->current_page = 0;
2156         dev_priv->sarea_priv->pfCurrentPage = dev_priv->current_page;
2157
2158         return 0;
2159 }
2160
2161 /* Called whenever a client dies, from drm_release.
2162  * NOTE:  Lock isn't necessarily held when this is called!
2163  */
2164 static int radeon_do_cleanup_pageflip(drm_device_t * dev)
2165 {
2166         drm_radeon_private_t *dev_priv = dev->dev_private;
2167         DRM_DEBUG("\n");
2168
2169         if (dev_priv->current_page != 0)
2170                 radeon_cp_dispatch_flip(dev);
2171
2172         dev_priv->page_flipping = 0;
2173         return 0;
2174 }
2175
2176 /* Swapping and flipping are different operations, need different ioctls.
2177  * They can & should be intermixed to support multiple 3d windows.
2178  */
2179 static int radeon_cp_flip(DRM_IOCTL_ARGS)
2180 {
2181         DRM_DEVICE;
2182         drm_radeon_private_t *dev_priv = dev->dev_private;
2183         DRM_DEBUG("\n");
2184
2185         LOCK_TEST_WITH_RETURN(dev, filp);
2186
2187         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2188
2189         if (!dev_priv->page_flipping)
2190                 radeon_do_init_pageflip(dev);
2191
2192         radeon_cp_dispatch_flip(dev);
2193
2194         COMMIT_RING();
2195         return 0;
2196 }
2197
2198 static int radeon_cp_swap(DRM_IOCTL_ARGS)
2199 {
2200         DRM_DEVICE;
2201         drm_radeon_private_t *dev_priv = dev->dev_private;
2202         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2203         DRM_DEBUG("\n");
2204
2205         LOCK_TEST_WITH_RETURN(dev, filp);
2206
2207         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2208
2209         if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
2210                 sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
2211
2212         radeon_cp_dispatch_swap(dev);
2213         dev_priv->sarea_priv->ctx_owner = 0;
2214
2215         COMMIT_RING();
2216         return 0;
2217 }
2218
2219 static int radeon_cp_vertex(DRM_IOCTL_ARGS)
2220 {
2221         DRM_DEVICE;
2222         drm_radeon_private_t *dev_priv = dev->dev_private;
2223         drm_file_t *filp_priv;
2224         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2225         drm_device_dma_t *dma = dev->dma;
2226         drm_buf_t *buf;
2227         drm_radeon_vertex_t vertex;
2228         drm_radeon_tcl_prim_t prim;
2229
2230         LOCK_TEST_WITH_RETURN(dev, filp);
2231
2232         DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
2233
2234         DRM_COPY_FROM_USER_IOCTL(vertex, (drm_radeon_vertex_t __user *) data,
2235                                  sizeof(vertex));
2236
2237         DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n",
2238                   DRM_CURRENTPID, vertex.idx, vertex.count, vertex.discard);
2239
2240         if (vertex.idx < 0 || vertex.idx >= dma->buf_count) {
2241                 DRM_ERROR("buffer index %d (of %d max)\n",
2242                           vertex.idx, dma->buf_count - 1);
2243                 return DRM_ERR(EINVAL);
2244         }
2245         if (vertex.prim < 0 || vertex.prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
2246                 DRM_ERROR("buffer prim %d\n", vertex.prim);
2247                 return DRM_ERR(EINVAL);
2248         }
2249
2250         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2251         VB_AGE_TEST_WITH_RETURN(dev_priv);
2252
2253         buf = dma->buflist[vertex.idx];
2254
2255         if (buf->filp != filp) {
2256                 DRM_ERROR("process %d using buffer owned by %p\n",
2257                           DRM_CURRENTPID, buf->filp);
2258                 return DRM_ERR(EINVAL);
2259         }
2260         if (buf->pending) {
2261                 DRM_ERROR("sending pending buffer %d\n", vertex.idx);
2262                 return DRM_ERR(EINVAL);
2263         }
2264
2265         /* Build up a prim_t record:
2266          */
2267         if (vertex.count) {
2268                 buf->used = vertex.count;       /* not used? */
2269
2270                 if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
2271                         if (radeon_emit_state(dev_priv, filp_priv,
2272                                               &sarea_priv->context_state,
2273                                               sarea_priv->tex_state,
2274                                               sarea_priv->dirty)) {
2275                                 DRM_ERROR("radeon_emit_state failed\n");
2276                                 return DRM_ERR(EINVAL);
2277                         }
2278
2279                         sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
2280                                                RADEON_UPLOAD_TEX1IMAGES |
2281                                                RADEON_UPLOAD_TEX2IMAGES |
2282                                                RADEON_REQUIRE_QUIESCENCE);
2283                 }
2284
2285                 prim.start = 0;
2286                 prim.finish = vertex.count;     /* unused */
2287                 prim.prim = vertex.prim;
2288                 prim.numverts = vertex.count;
2289                 prim.vc_format = dev_priv->sarea_priv->vc_format;
2290
2291                 radeon_cp_dispatch_vertex(dev, buf, &prim);
2292         }
2293
2294         if (vertex.discard) {
2295                 radeon_cp_discard_buffer(dev, buf);
2296         }
2297
2298         COMMIT_RING();
2299         return 0;
2300 }
2301
2302 static int radeon_cp_indices(DRM_IOCTL_ARGS)
2303 {
2304         DRM_DEVICE;
2305         drm_radeon_private_t *dev_priv = dev->dev_private;
2306         drm_file_t *filp_priv;
2307         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2308         drm_device_dma_t *dma = dev->dma;
2309         drm_buf_t *buf;
2310         drm_radeon_indices_t elts;
2311         drm_radeon_tcl_prim_t prim;
2312         int count;
2313
2314         LOCK_TEST_WITH_RETURN(dev, filp);
2315
2316         DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
2317
2318         DRM_COPY_FROM_USER_IOCTL(elts, (drm_radeon_indices_t __user *) data,
2319                                  sizeof(elts));
2320
2321         DRM_DEBUG("pid=%d index=%d start=%d end=%d discard=%d\n",
2322                   DRM_CURRENTPID, elts.idx, elts.start, elts.end, elts.discard);
2323
2324         if (elts.idx < 0 || elts.idx >= dma->buf_count) {
2325                 DRM_ERROR("buffer index %d (of %d max)\n",
2326                           elts.idx, dma->buf_count - 1);
2327                 return DRM_ERR(EINVAL);
2328         }
2329         if (elts.prim < 0 || elts.prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
2330                 DRM_ERROR("buffer prim %d\n", elts.prim);
2331                 return DRM_ERR(EINVAL);
2332         }
2333
2334         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2335         VB_AGE_TEST_WITH_RETURN(dev_priv);
2336
2337         buf = dma->buflist[elts.idx];
2338
2339         if (buf->filp != filp) {
2340                 DRM_ERROR("process %d using buffer owned by %p\n",
2341                           DRM_CURRENTPID, buf->filp);
2342                 return DRM_ERR(EINVAL);
2343         }
2344         if (buf->pending) {
2345                 DRM_ERROR("sending pending buffer %d\n", elts.idx);
2346                 return DRM_ERR(EINVAL);
2347         }
2348
2349         count = (elts.end - elts.start) / sizeof(u16);
2350         elts.start -= RADEON_INDEX_PRIM_OFFSET;
2351
2352         if (elts.start & 0x7) {
2353                 DRM_ERROR("misaligned buffer 0x%x\n", elts.start);
2354                 return DRM_ERR(EINVAL);
2355         }
2356         if (elts.start < buf->used) {
2357                 DRM_ERROR("no header 0x%x - 0x%x\n", elts.start, buf->used);
2358                 return DRM_ERR(EINVAL);
2359         }
2360
2361         buf->used = elts.end;
2362
2363         if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
2364                 if (radeon_emit_state(dev_priv, filp_priv,
2365                                       &sarea_priv->context_state,
2366                                       sarea_priv->tex_state,
2367                                       sarea_priv->dirty)) {
2368                         DRM_ERROR("radeon_emit_state failed\n");
2369                         return DRM_ERR(EINVAL);
2370                 }
2371
2372                 sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
2373                                        RADEON_UPLOAD_TEX1IMAGES |
2374                                        RADEON_UPLOAD_TEX2IMAGES |
2375                                        RADEON_REQUIRE_QUIESCENCE);
2376         }
2377
2378         /* Build up a prim_t record:
2379          */
2380         prim.start = elts.start;
2381         prim.finish = elts.end;
2382         prim.prim = elts.prim;
2383         prim.offset = 0;        /* offset from start of dma buffers */
2384         prim.numverts = RADEON_MAX_VB_VERTS;    /* duh */
2385         prim.vc_format = dev_priv->sarea_priv->vc_format;
2386
2387         radeon_cp_dispatch_indices(dev, buf, &prim);
2388         if (elts.discard) {
2389                 radeon_cp_discard_buffer(dev, buf);
2390         }
2391
2392         COMMIT_RING();
2393         return 0;
2394 }
2395
2396 static int radeon_cp_texture(DRM_IOCTL_ARGS)
2397 {
2398         DRM_DEVICE;
2399         drm_radeon_private_t *dev_priv = dev->dev_private;
2400         drm_radeon_texture_t tex;
2401         drm_radeon_tex_image_t image;
2402         int ret;
2403
2404         LOCK_TEST_WITH_RETURN(dev, filp);
2405
2406         DRM_COPY_FROM_USER_IOCTL(tex, (drm_radeon_texture_t __user *) data,
2407                                  sizeof(tex));
2408
2409         if (tex.image == NULL) {
2410                 DRM_ERROR("null texture image!\n");
2411                 return DRM_ERR(EINVAL);
2412         }
2413
2414         if (DRM_COPY_FROM_USER(&image,
2415                                (drm_radeon_tex_image_t __user *) tex.image,
2416                                sizeof(image)))
2417                 return DRM_ERR(EFAULT);
2418
2419         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2420         VB_AGE_TEST_WITH_RETURN(dev_priv);
2421
2422         ret = radeon_cp_dispatch_texture(filp, dev, &tex, &image);
2423
2424         COMMIT_RING();
2425         return ret;
2426 }
2427
2428 static int radeon_cp_stipple(DRM_IOCTL_ARGS)
2429 {
2430         DRM_DEVICE;
2431         drm_radeon_private_t *dev_priv = dev->dev_private;
2432         drm_radeon_stipple_t stipple;
2433         u32 mask[32];
2434
2435         LOCK_TEST_WITH_RETURN(dev, filp);
2436
2437         DRM_COPY_FROM_USER_IOCTL(stipple, (drm_radeon_stipple_t __user *) data,
2438                                  sizeof(stipple));
2439
2440         if (DRM_COPY_FROM_USER(&mask, stipple.mask, 32 * sizeof(u32)))
2441                 return DRM_ERR(EFAULT);
2442
2443         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2444
2445         radeon_cp_dispatch_stipple(dev, mask);
2446
2447         COMMIT_RING();
2448         return 0;
2449 }
2450
2451 static int radeon_cp_indirect(DRM_IOCTL_ARGS)
2452 {
2453         DRM_DEVICE;
2454         drm_radeon_private_t *dev_priv = dev->dev_private;
2455         drm_device_dma_t *dma = dev->dma;
2456         drm_buf_t *buf;
2457         drm_radeon_indirect_t indirect;
2458         RING_LOCALS;
2459
2460         LOCK_TEST_WITH_RETURN(dev, filp);
2461
2462         DRM_COPY_FROM_USER_IOCTL(indirect,
2463                                  (drm_radeon_indirect_t __user *) data,
2464                                  sizeof(indirect));
2465
2466         DRM_DEBUG("indirect: idx=%d s=%d e=%d d=%d\n",
2467                   indirect.idx, indirect.start, indirect.end, indirect.discard);
2468
2469         if (indirect.idx < 0 || indirect.idx >= dma->buf_count) {
2470                 DRM_ERROR("buffer index %d (of %d max)\n",
2471                           indirect.idx, dma->buf_count - 1);
2472                 return DRM_ERR(EINVAL);
2473         }
2474
2475         buf = dma->buflist[indirect.idx];
2476
2477         if (buf->filp != filp) {
2478                 DRM_ERROR("process %d using buffer owned by %p\n",
2479                           DRM_CURRENTPID, buf->filp);
2480                 return DRM_ERR(EINVAL);
2481         }
2482         if (buf->pending) {
2483                 DRM_ERROR("sending pending buffer %d\n", indirect.idx);
2484                 return DRM_ERR(EINVAL);
2485         }
2486
2487         if (indirect.start < buf->used) {
2488                 DRM_ERROR("reusing indirect: start=0x%x actual=0x%x\n",
2489                           indirect.start, buf->used);
2490                 return DRM_ERR(EINVAL);
2491         }
2492
2493         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2494         VB_AGE_TEST_WITH_RETURN(dev_priv);
2495
2496         buf->used = indirect.end;
2497
2498         /* Wait for the 3D stream to idle before the indirect buffer
2499          * containing 2D acceleration commands is processed.
2500          */
2501         BEGIN_RING(2);
2502
2503         RADEON_WAIT_UNTIL_3D_IDLE();
2504
2505         ADVANCE_RING();
2506
2507         /* Dispatch the indirect buffer full of commands from the
2508          * X server.  This is insecure and is thus only available to
2509          * privileged clients.
2510          */
2511         radeon_cp_dispatch_indirect(dev, buf, indirect.start, indirect.end);
2512         if (indirect.discard) {
2513                 radeon_cp_discard_buffer(dev, buf);
2514         }
2515
2516         COMMIT_RING();
2517         return 0;
2518 }
2519
2520 static int radeon_cp_vertex2(DRM_IOCTL_ARGS)
2521 {
2522         DRM_DEVICE;
2523         drm_radeon_private_t *dev_priv = dev->dev_private;
2524         drm_file_t *filp_priv;
2525         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2526         drm_device_dma_t *dma = dev->dma;
2527         drm_buf_t *buf;
2528         drm_radeon_vertex2_t vertex;
2529         int i;
2530         unsigned char laststate;
2531
2532         LOCK_TEST_WITH_RETURN(dev, filp);
2533
2534         DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
2535
2536         DRM_COPY_FROM_USER_IOCTL(vertex, (drm_radeon_vertex2_t __user *) data,
2537                                  sizeof(vertex));
2538
2539         DRM_DEBUG("pid=%d index=%d discard=%d\n",
2540                   DRM_CURRENTPID, vertex.idx, vertex.discard);
2541
2542         if (vertex.idx < 0 || vertex.idx >= dma->buf_count) {
2543                 DRM_ERROR("buffer index %d (of %d max)\n",
2544                           vertex.idx, dma->buf_count - 1);
2545                 return DRM_ERR(EINVAL);
2546         }
2547
2548         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2549         VB_AGE_TEST_WITH_RETURN(dev_priv);
2550
2551         buf = dma->buflist[vertex.idx];
2552
2553         if (buf->filp != filp) {
2554                 DRM_ERROR("process %d using buffer owned by %p\n",
2555                           DRM_CURRENTPID, buf->filp);
2556                 return DRM_ERR(EINVAL);
2557         }
2558
2559         if (buf->pending) {
2560                 DRM_ERROR("sending pending buffer %d\n", vertex.idx);
2561                 return DRM_ERR(EINVAL);
2562         }
2563
2564         if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
2565                 return DRM_ERR(EINVAL);
2566
2567         for (laststate = 0xff, i = 0; i < vertex.nr_prims; i++) {
2568                 drm_radeon_prim_t prim;
2569                 drm_radeon_tcl_prim_t tclprim;
2570
2571                 if (DRM_COPY_FROM_USER(&prim, &vertex.prim[i], sizeof(prim)))
2572                         return DRM_ERR(EFAULT);
2573
2574                 if (prim.stateidx != laststate) {
2575                         drm_radeon_state_t state;
2576
2577                         if (DRM_COPY_FROM_USER(&state,
2578                                                &vertex.state[prim.stateidx],
2579                                                sizeof(state)))
2580                                 return DRM_ERR(EFAULT);
2581
2582                         if (radeon_emit_state2(dev_priv, filp_priv, &state)) {
2583                                 DRM_ERROR("radeon_emit_state2 failed\n");
2584                                 return DRM_ERR(EINVAL);
2585                         }
2586
2587                         laststate = prim.stateidx;
2588                 }
2589
2590                 tclprim.start = prim.start;
2591                 tclprim.finish = prim.finish;
2592                 tclprim.prim = prim.prim;
2593                 tclprim.vc_format = prim.vc_format;
2594
2595                 if (prim.prim & RADEON_PRIM_WALK_IND) {
2596                         tclprim.offset = prim.numverts * 64;
2597                         tclprim.numverts = RADEON_MAX_VB_VERTS; /* duh */
2598
2599                         radeon_cp_dispatch_indices(dev, buf, &tclprim);
2600                 } else {
2601                         tclprim.numverts = prim.numverts;
2602                         tclprim.offset = 0;     /* not used */
2603
2604                         radeon_cp_dispatch_vertex(dev, buf, &tclprim);
2605                 }
2606
2607                 if (sarea_priv->nbox == 1)
2608                         sarea_priv->nbox = 0;
2609         }
2610
2611         if (vertex.discard) {
2612                 radeon_cp_discard_buffer(dev, buf);
2613         }
2614
2615         COMMIT_RING();
2616         return 0;
2617 }
2618
2619 static int radeon_emit_packets(drm_radeon_private_t * dev_priv,
2620                                drm_file_t * filp_priv,
2621                                drm_radeon_cmd_header_t header,
2622                                drm_radeon_kcmd_buffer_t *cmdbuf)
2623 {
2624         int id = (int)header.packet.packet_id;
2625         int sz, reg;
2626         int *data = (int *)cmdbuf->buf;
2627         RING_LOCALS;
2628
2629         if (id >= RADEON_MAX_STATE_PACKETS)
2630                 return DRM_ERR(EINVAL);
2631
2632         sz = packet[id].len;
2633         reg = packet[id].start;
2634
2635         if (sz * sizeof(int) > cmdbuf->bufsz) {
2636                 DRM_ERROR("Packet size provided larger than data provided\n");
2637                 return DRM_ERR(EINVAL);
2638         }
2639
2640         if (radeon_check_and_fixup_packets(dev_priv, filp_priv, id, data)) {
2641                 DRM_ERROR("Packet verification failed\n");
2642                 return DRM_ERR(EINVAL);
2643         }
2644
2645         BEGIN_RING(sz + 1);
2646         OUT_RING(CP_PACKET0(reg, (sz - 1)));
2647         OUT_RING_TABLE(data, sz);
2648         ADVANCE_RING();
2649
2650         cmdbuf->buf += sz * sizeof(int);
2651         cmdbuf->bufsz -= sz * sizeof(int);
2652         return 0;
2653 }
2654
2655 static __inline__ int radeon_emit_scalars(drm_radeon_private_t *dev_priv,
2656                                           drm_radeon_cmd_header_t header,
2657                                           drm_radeon_kcmd_buffer_t *cmdbuf)
2658 {
2659         int sz = header.scalars.count;
2660         int start = header.scalars.offset;
2661         int stride = header.scalars.stride;
2662         RING_LOCALS;
2663
2664         BEGIN_RING(3 + sz);
2665         OUT_RING(CP_PACKET0(RADEON_SE_TCL_SCALAR_INDX_REG, 0));
2666         OUT_RING(start | (stride << RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT));
2667         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_SCALAR_DATA_REG, sz - 1));
2668         OUT_RING_TABLE(cmdbuf->buf, sz);
2669         ADVANCE_RING();
2670         cmdbuf->buf += sz * sizeof(int);
2671         cmdbuf->bufsz -= sz * sizeof(int);
2672         return 0;
2673 }
2674
2675 /* God this is ugly
2676  */
2677 static __inline__ int radeon_emit_scalars2(drm_radeon_private_t *dev_priv,
2678                                            drm_radeon_cmd_header_t header,
2679                                            drm_radeon_kcmd_buffer_t *cmdbuf)
2680 {
2681         int sz = header.scalars.count;
2682         int start = ((unsigned int)header.scalars.offset) + 0x100;
2683         int stride = header.scalars.stride;
2684         RING_LOCALS;
2685
2686         BEGIN_RING(3 + sz);
2687         OUT_RING(CP_PACKET0(RADEON_SE_TCL_SCALAR_INDX_REG, 0));
2688         OUT_RING(start | (stride << RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT));
2689         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_SCALAR_DATA_REG, sz - 1));
2690         OUT_RING_TABLE(cmdbuf->buf, sz);
2691         ADVANCE_RING();
2692         cmdbuf->buf += sz * sizeof(int);
2693         cmdbuf->bufsz -= sz * sizeof(int);
2694         return 0;
2695 }
2696
2697 static __inline__ int radeon_emit_vectors(drm_radeon_private_t *dev_priv,
2698                                           drm_radeon_cmd_header_t header,
2699                                           drm_radeon_kcmd_buffer_t *cmdbuf)
2700 {
2701         int sz = header.vectors.count;
2702         int start = header.vectors.offset;
2703         int stride = header.vectors.stride;
2704         RING_LOCALS;
2705
2706         BEGIN_RING(5 + sz);
2707         OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
2708         OUT_RING(CP_PACKET0(RADEON_SE_TCL_VECTOR_INDX_REG, 0));
2709         OUT_RING(start | (stride << RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT));
2710         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_VECTOR_DATA_REG, (sz - 1)));
2711         OUT_RING_TABLE(cmdbuf->buf, sz);
2712         ADVANCE_RING();
2713
2714         cmdbuf->buf += sz * sizeof(int);
2715         cmdbuf->bufsz -= sz * sizeof(int);
2716         return 0;
2717 }
2718
2719 static __inline__ int radeon_emit_veclinear(drm_radeon_private_t *dev_priv,
2720                                           drm_radeon_cmd_header_t header,
2721                                           drm_radeon_kcmd_buffer_t *cmdbuf)
2722 {
2723         int sz = header.veclinear.count * 4;
2724         int start = header.veclinear.addr_lo | (header.veclinear.addr_hi << 8);
2725         RING_LOCALS;
2726
2727         if (!sz)
2728                 return 0;
2729         if (sz * 4 > cmdbuf->bufsz)
2730                 return DRM_ERR(EINVAL);
2731
2732         BEGIN_RING(5 + sz);
2733         OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
2734         OUT_RING(CP_PACKET0(RADEON_SE_TCL_VECTOR_INDX_REG, 0));
2735         OUT_RING(start | (1 << RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT));
2736         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_VECTOR_DATA_REG, (sz - 1)));
2737         OUT_RING_TABLE(cmdbuf->buf, sz);
2738         ADVANCE_RING();
2739
2740         cmdbuf->buf += sz * sizeof(int);
2741         cmdbuf->bufsz -= sz * sizeof(int);
2742         return 0;
2743 }
2744
2745 static int radeon_emit_packet3(drm_device_t * dev,
2746                                drm_file_t * filp_priv,
2747                                drm_radeon_kcmd_buffer_t *cmdbuf)
2748 {
2749         drm_radeon_private_t *dev_priv = dev->dev_private;
2750         unsigned int cmdsz;
2751         int ret;
2752         RING_LOCALS;
2753
2754         DRM_DEBUG("\n");
2755
2756         if ((ret = radeon_check_and_fixup_packet3(dev_priv, filp_priv,
2757                                                   cmdbuf, &cmdsz))) {
2758                 DRM_ERROR("Packet verification failed\n");
2759                 return ret;
2760         }
2761
2762         BEGIN_RING(cmdsz);
2763         OUT_RING_TABLE(cmdbuf->buf, cmdsz);
2764         ADVANCE_RING();
2765
2766         cmdbuf->buf += cmdsz * 4;
2767         cmdbuf->bufsz -= cmdsz * 4;
2768         return 0;
2769 }
2770
2771 static int radeon_emit_packet3_cliprect(drm_device_t *dev,
2772                                         drm_file_t *filp_priv,
2773                                         drm_radeon_kcmd_buffer_t *cmdbuf,
2774                                         int orig_nbox)
2775 {
2776         drm_radeon_private_t *dev_priv = dev->dev_private;
2777         drm_clip_rect_t box;
2778         unsigned int cmdsz;
2779         int ret;
2780         drm_clip_rect_t __user *boxes = cmdbuf->boxes;
2781         int i = 0;
2782         RING_LOCALS;
2783
2784         DRM_DEBUG("\n");
2785
2786         if ((ret = radeon_check_and_fixup_packet3(dev_priv, filp_priv,
2787                                                   cmdbuf, &cmdsz))) {
2788                 DRM_ERROR("Packet verification failed\n");
2789                 return ret;
2790         }
2791
2792         if (!orig_nbox)
2793                 goto out;
2794
2795         do {
2796                 if (i < cmdbuf->nbox) {
2797                         if (DRM_COPY_FROM_USER(&box, &boxes[i], sizeof(box)))
2798                                 return DRM_ERR(EFAULT);
2799                         /* FIXME The second and subsequent times round
2800                          * this loop, send a WAIT_UNTIL_3D_IDLE before
2801                          * calling emit_clip_rect(). This fixes a
2802                          * lockup on fast machines when sending
2803                          * several cliprects with a cmdbuf, as when
2804                          * waving a 2D window over a 3D
2805                          * window. Something in the commands from user
2806                          * space seems to hang the card when they're
2807                          * sent several times in a row. That would be
2808                          * the correct place to fix it but this works
2809                          * around it until I can figure that out - Tim
2810                          * Smith */
2811                         if (i) {
2812                                 BEGIN_RING(2);
2813                                 RADEON_WAIT_UNTIL_3D_IDLE();
2814                                 ADVANCE_RING();
2815                         }
2816                         radeon_emit_clip_rect(dev_priv, &box);
2817                 }
2818
2819                 BEGIN_RING(cmdsz);
2820                 OUT_RING_TABLE(cmdbuf->buf, cmdsz);
2821                 ADVANCE_RING();
2822
2823         } while (++i < cmdbuf->nbox);
2824         if (cmdbuf->nbox == 1)
2825                 cmdbuf->nbox = 0;
2826
2827       out:
2828         cmdbuf->buf += cmdsz * 4;
2829         cmdbuf->bufsz -= cmdsz * 4;
2830         return 0;
2831 }
2832
2833 static int radeon_emit_wait(drm_device_t * dev, int flags)
2834 {
2835         drm_radeon_private_t *dev_priv = dev->dev_private;
2836         RING_LOCALS;
2837
2838         DRM_DEBUG("%s: %x\n", __FUNCTION__, flags);
2839         switch (flags) {
2840         case RADEON_WAIT_2D:
2841                 BEGIN_RING(2);
2842                 RADEON_WAIT_UNTIL_2D_IDLE();
2843                 ADVANCE_RING();
2844                 break;
2845         case RADEON_WAIT_3D:
2846                 BEGIN_RING(2);
2847                 RADEON_WAIT_UNTIL_3D_IDLE();
2848                 ADVANCE_RING();
2849                 break;
2850         case RADEON_WAIT_2D | RADEON_WAIT_3D:
2851                 BEGIN_RING(2);
2852                 RADEON_WAIT_UNTIL_IDLE();
2853                 ADVANCE_RING();
2854                 break;
2855         default:
2856                 return DRM_ERR(EINVAL);
2857         }
2858
2859         return 0;
2860 }
2861
2862 static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
2863 {
2864         DRM_DEVICE;
2865         drm_radeon_private_t *dev_priv = dev->dev_private;
2866         drm_file_t *filp_priv;
2867         drm_device_dma_t *dma = dev->dma;
2868         drm_buf_t *buf = NULL;
2869         int idx;
2870         drm_radeon_kcmd_buffer_t cmdbuf;
2871         drm_radeon_cmd_header_t header;
2872         int orig_nbox, orig_bufsz;
2873         char *kbuf = NULL;
2874
2875         LOCK_TEST_WITH_RETURN(dev, filp);
2876
2877         DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
2878
2879         DRM_COPY_FROM_USER_IOCTL(cmdbuf,
2880                                  (drm_radeon_cmd_buffer_t __user *) data,
2881                                  sizeof(cmdbuf));
2882
2883         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2884         VB_AGE_TEST_WITH_RETURN(dev_priv);
2885
2886         if (cmdbuf.bufsz > 64 * 1024 || cmdbuf.bufsz < 0) {
2887                 return DRM_ERR(EINVAL);
2888         }
2889
2890         /* Allocate an in-kernel area and copy in the cmdbuf.  Do this to avoid
2891          * races between checking values and using those values in other code,
2892          * and simply to avoid a lot of function calls to copy in data.
2893          */
2894         orig_bufsz = cmdbuf.bufsz;
2895         if (orig_bufsz != 0) {
2896                 kbuf = drm_alloc(cmdbuf.bufsz, DRM_MEM_DRIVER);
2897                 if (kbuf == NULL)
2898                         return DRM_ERR(ENOMEM);
2899                 if (DRM_COPY_FROM_USER(kbuf, (void __user *)cmdbuf.buf,
2900                                        cmdbuf.bufsz)) {
2901                         drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
2902                         return DRM_ERR(EFAULT);
2903                 }
2904                 cmdbuf.buf = kbuf;
2905         }
2906
2907         orig_nbox = cmdbuf.nbox;
2908
2909         if (dev_priv->microcode_version == UCODE_R300) {
2910                 int temp;
2911                 temp = r300_do_cp_cmdbuf(dev, filp, filp_priv, &cmdbuf);
2912
2913                 if (orig_bufsz != 0)
2914                         drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
2915
2916                 return temp;
2917         }
2918
2919         /* microcode_version != r300 */
2920         while (cmdbuf.bufsz >= sizeof(header)) {
2921
2922                 header.i = *(int *)cmdbuf.buf;
2923                 cmdbuf.buf += sizeof(header);
2924                 cmdbuf.bufsz -= sizeof(header);
2925
2926                 switch (header.header.cmd_type) {
2927                 case RADEON_CMD_PACKET:
2928                         DRM_DEBUG("RADEON_CMD_PACKET\n");
2929                         if (radeon_emit_packets
2930                             (dev_priv, filp_priv, header, &cmdbuf)) {
2931                                 DRM_ERROR("radeon_emit_packets failed\n");
2932                                 goto err;
2933                         }
2934                         break;
2935
2936                 case RADEON_CMD_SCALARS:
2937                         DRM_DEBUG("RADEON_CMD_SCALARS\n");
2938                         if (radeon_emit_scalars(dev_priv, header, &cmdbuf)) {
2939                                 DRM_ERROR("radeon_emit_scalars failed\n");
2940                                 goto err;
2941                         }
2942                         break;
2943
2944                 case RADEON_CMD_VECTORS:
2945                         DRM_DEBUG("RADEON_CMD_VECTORS\n");
2946                         if (radeon_emit_vectors(dev_priv, header, &cmdbuf)) {
2947                                 DRM_ERROR("radeon_emit_vectors failed\n");
2948                                 goto err;
2949                         }
2950                         break;
2951
2952                 case RADEON_CMD_DMA_DISCARD:
2953                         DRM_DEBUG("RADEON_CMD_DMA_DISCARD\n");
2954                         idx = header.dma.buf_idx;
2955                         if (idx < 0 || idx >= dma->buf_count) {
2956                                 DRM_ERROR("buffer index %d (of %d max)\n",
2957                                           idx, dma->buf_count - 1);
2958                                 goto err;
2959                         }
2960
2961                         buf = dma->buflist[idx];
2962                         if (buf->filp != filp || buf->pending) {
2963                                 DRM_ERROR("bad buffer %p %p %d\n",
2964                                           buf->filp, filp, buf->pending);
2965                                 goto err;
2966                         }
2967
2968                         radeon_cp_discard_buffer(dev, buf);
2969                         break;
2970
2971                 case RADEON_CMD_PACKET3:
2972                         DRM_DEBUG("RADEON_CMD_PACKET3\n");
2973                         if (radeon_emit_packet3(dev, filp_priv, &cmdbuf)) {
2974                                 DRM_ERROR("radeon_emit_packet3 failed\n");
2975                                 goto err;
2976                         }
2977                         break;
2978
2979                 case RADEON_CMD_PACKET3_CLIP:
2980                         DRM_DEBUG("RADEON_CMD_PACKET3_CLIP\n");
2981                         if (radeon_emit_packet3_cliprect
2982                             (dev, filp_priv, &cmdbuf, orig_nbox)) {
2983                                 DRM_ERROR("radeon_emit_packet3_clip failed\n");
2984                                 goto err;
2985                         }
2986                         break;
2987
2988                 case RADEON_CMD_SCALARS2:
2989                         DRM_DEBUG("RADEON_CMD_SCALARS2\n");
2990                         if (radeon_emit_scalars2(dev_priv, header, &cmdbuf)) {
2991                                 DRM_ERROR("radeon_emit_scalars2 failed\n");
2992                                 goto err;
2993                         }
2994                         break;
2995
2996                 case RADEON_CMD_WAIT:
2997                         DRM_DEBUG("RADEON_CMD_WAIT\n");
2998                         if (radeon_emit_wait(dev, header.wait.flags)) {
2999                                 DRM_ERROR("radeon_emit_wait failed\n");
3000                                 goto err;
3001                         }
3002                         break;
3003                 case RADEON_CMD_VECLINEAR:
3004                         DRM_DEBUG("RADEON_CMD_VECLINEAR\n");
3005                         if (radeon_emit_veclinear(dev_priv, header, &cmdbuf)) {
3006                                 DRM_ERROR("radeon_emit_veclinear failed\n");
3007                                 goto err;
3008                         }
3009                         break;
3010
3011                 default:
3012                         DRM_ERROR("bad cmd_type %d at %p\n",
3013                                   header.header.cmd_type,
3014                                   cmdbuf.buf - sizeof(header));
3015                         goto err;
3016                 }
3017         }
3018
3019         if (orig_bufsz != 0)
3020                 drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
3021
3022         DRM_DEBUG("DONE\n");
3023         COMMIT_RING();
3024         return 0;
3025
3026       err:
3027         if (orig_bufsz != 0)
3028                 drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
3029         return DRM_ERR(EINVAL);
3030 }
3031
3032 static int radeon_cp_getparam(DRM_IOCTL_ARGS)
3033 {
3034         DRM_DEVICE;
3035         drm_radeon_private_t *dev_priv = dev->dev_private;
3036         drm_radeon_getparam_t param;
3037         int value;
3038
3039         DRM_COPY_FROM_USER_IOCTL(param, (drm_radeon_getparam_t __user *) data,
3040                                  sizeof(param));
3041
3042         DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
3043
3044         switch (param.param) {
3045         case RADEON_PARAM_GART_BUFFER_OFFSET:
3046                 value = dev_priv->gart_buffers_offset;
3047                 break;
3048         case RADEON_PARAM_LAST_FRAME:
3049                 dev_priv->stats.last_frame_reads++;
3050                 value = GET_SCRATCH(0);
3051                 break;
3052         case RADEON_PARAM_LAST_DISPATCH:
3053                 value = GET_SCRATCH(1);
3054                 break;
3055         case RADEON_PARAM_LAST_CLEAR:
3056                 dev_priv->stats.last_clear_reads++;
3057                 value = GET_SCRATCH(2);
3058                 break;
3059         case RADEON_PARAM_IRQ_NR:
3060                 value = dev->irq;
3061                 break;
3062         case RADEON_PARAM_GART_BASE:
3063                 value = dev_priv->gart_vm_start;
3064                 break;
3065         case RADEON_PARAM_REGISTER_HANDLE:
3066                 value = dev_priv->mmio->offset;
3067                 break;
3068         case RADEON_PARAM_STATUS_HANDLE:
3069                 value = dev_priv->ring_rptr_offset;
3070                 break;
3071 #if BITS_PER_LONG == 32
3072                 /*
3073                  * This ioctl() doesn't work on 64-bit platforms because hw_lock is a
3074                  * pointer which can't fit into an int-sized variable.  According to
3075                  * Michel Dänzer, the ioctl() is only used on embedded platforms, so
3076                  * not supporting it shouldn't be a problem.  If the same functionality
3077                  * is needed on 64-bit platforms, a new ioctl() would have to be added,
3078                  * so backwards-compatibility for the embedded platforms can be
3079                  * maintained.  --davidm 4-Feb-2004.
3080                  */
3081         case RADEON_PARAM_SAREA_HANDLE:
3082                 /* The lock is the first dword in the sarea. */
3083                 value = (long)dev->lock.hw_lock;
3084                 break;
3085 #endif
3086         case RADEON_PARAM_GART_TEX_HANDLE:
3087                 value = dev_priv->gart_textures_offset;
3088                 break;
3089         case RADEON_PARAM_SCRATCH_OFFSET:
3090                 if (!dev_priv->writeback_works)
3091                         return DRM_ERR(EINVAL);
3092                 value = RADEON_SCRATCH_REG_OFFSET;
3093                 break;
3094         case RADEON_PARAM_CARD_TYPE:
3095                 if (dev_priv->flags & RADEON_IS_PCIE)
3096                         value = RADEON_CARD_PCIE;
3097                 else if (dev_priv->flags & RADEON_IS_AGP)
3098                         value = RADEON_CARD_AGP;
3099                 else
3100                         value = RADEON_CARD_PCI;
3101                 break;
3102         default:
3103                 DRM_DEBUG("Invalid parameter %d\n", param.param);
3104                 return DRM_ERR(EINVAL);
3105         }
3106
3107         if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
3108                 DRM_ERROR("copy_to_user\n");
3109                 return DRM_ERR(EFAULT);
3110         }
3111
3112         return 0;
3113 }
3114
3115 static int radeon_cp_setparam(DRM_IOCTL_ARGS)
3116 {
3117         DRM_DEVICE;
3118         drm_radeon_private_t *dev_priv = dev->dev_private;
3119         drm_file_t *filp_priv;
3120         drm_radeon_setparam_t sp;
3121         struct drm_radeon_driver_file_fields *radeon_priv;
3122
3123         DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
3124
3125         DRM_COPY_FROM_USER_IOCTL(sp, (drm_radeon_setparam_t __user *) data,
3126                                  sizeof(sp));
3127
3128         switch (sp.param) {
3129         case RADEON_SETPARAM_FB_LOCATION:
3130                 radeon_priv = filp_priv->driver_priv;
3131                 radeon_priv->radeon_fb_delta = dev_priv->fb_location - sp.value;
3132                 break;
3133         case RADEON_SETPARAM_SWITCH_TILING:
3134                 if (sp.value == 0) {
3135                         DRM_DEBUG("color tiling disabled\n");
3136                         dev_priv->front_pitch_offset &= ~RADEON_DST_TILE_MACRO;
3137                         dev_priv->back_pitch_offset &= ~RADEON_DST_TILE_MACRO;
3138                         dev_priv->sarea_priv->tiling_enabled = 0;
3139                 } else if (sp.value == 1) {
3140                         DRM_DEBUG("color tiling enabled\n");
3141                         dev_priv->front_pitch_offset |= RADEON_DST_TILE_MACRO;
3142                         dev_priv->back_pitch_offset |= RADEON_DST_TILE_MACRO;
3143                         dev_priv->sarea_priv->tiling_enabled = 1;
3144                 }
3145                 break;
3146         case RADEON_SETPARAM_PCIGART_LOCATION:
3147                 dev_priv->pcigart_offset = sp.value;
3148                 break;
3149         case RADEON_SETPARAM_NEW_MEMMAP:
3150                 dev_priv->new_memmap = sp.value;
3151                 break;
3152         default:
3153                 DRM_DEBUG("Invalid parameter %d\n", sp.param);
3154                 return DRM_ERR(EINVAL);
3155         }
3156
3157         return 0;
3158 }
3159
3160 /* When a client dies:
3161  *    - Check for and clean up flipped page state
3162  *    - Free any alloced GART memory.
3163  *    - Free any alloced radeon surfaces.
3164  *
3165  * DRM infrastructure takes care of reclaiming dma buffers.
3166  */
3167 void radeon_driver_preclose(drm_device_t * dev, DRMFILE filp)
3168 {
3169         if (dev->dev_private) {
3170                 drm_radeon_private_t *dev_priv = dev->dev_private;
3171                 if (dev_priv->page_flipping) {
3172                         radeon_do_cleanup_pageflip(dev);
3173                 }
3174                 radeon_mem_release(filp, dev_priv->gart_heap);
3175                 radeon_mem_release(filp, dev_priv->fb_heap);
3176                 radeon_surfaces_release(filp, dev_priv);
3177         }
3178 }
3179
3180 void radeon_driver_lastclose(drm_device_t * dev)
3181 {
3182         radeon_do_release(dev);
3183 }
3184
3185 int radeon_driver_open(drm_device_t * dev, drm_file_t * filp_priv)
3186 {
3187         drm_radeon_private_t *dev_priv = dev->dev_private;
3188         struct drm_radeon_driver_file_fields *radeon_priv;
3189
3190         DRM_DEBUG("\n");
3191         radeon_priv =
3192             (struct drm_radeon_driver_file_fields *)
3193             drm_alloc(sizeof(*radeon_priv), DRM_MEM_FILES);
3194
3195         if (!radeon_priv)
3196                 return -ENOMEM;
3197
3198         filp_priv->driver_priv = radeon_priv;
3199
3200         if (dev_priv)
3201                 radeon_priv->radeon_fb_delta = dev_priv->fb_location;
3202         else
3203                 radeon_priv->radeon_fb_delta = 0;
3204         return 0;
3205 }
3206
3207 void radeon_driver_postclose(drm_device_t * dev, drm_file_t * filp_priv)
3208 {
3209         struct drm_radeon_driver_file_fields *radeon_priv =
3210             filp_priv->driver_priv;
3211
3212         drm_free(radeon_priv, sizeof(*radeon_priv), DRM_MEM_FILES);
3213 }
3214
3215 drm_ioctl_desc_t radeon_ioctls[] = {
3216         [DRM_IOCTL_NR(DRM_RADEON_CP_INIT)] = {radeon_cp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
3217         [DRM_IOCTL_NR(DRM_RADEON_CP_START)] = {radeon_cp_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
3218         [DRM_IOCTL_NR(DRM_RADEON_CP_STOP)] = {radeon_cp_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
3219         [DRM_IOCTL_NR(DRM_RADEON_CP_RESET)] = {radeon_cp_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
3220         [DRM_IOCTL_NR(DRM_RADEON_CP_IDLE)] = {radeon_cp_idle, DRM_AUTH},
3221         [DRM_IOCTL_NR(DRM_RADEON_CP_RESUME)] = {radeon_cp_resume, DRM_AUTH},
3222         [DRM_IOCTL_NR(DRM_RADEON_RESET)] = {radeon_engine_reset, DRM_AUTH},
3223         [DRM_IOCTL_NR(DRM_RADEON_FULLSCREEN)] = {radeon_fullscreen, DRM_AUTH},
3224         [DRM_IOCTL_NR(DRM_RADEON_SWAP)] = {radeon_cp_swap, DRM_AUTH},
3225         [DRM_IOCTL_NR(DRM_RADEON_CLEAR)] = {radeon_cp_clear, DRM_AUTH},
3226         [DRM_IOCTL_NR(DRM_RADEON_VERTEX)] = {radeon_cp_vertex, DRM_AUTH},
3227         [DRM_IOCTL_NR(DRM_RADEON_INDICES)] = {radeon_cp_indices, DRM_AUTH},
3228         [DRM_IOCTL_NR(DRM_RADEON_TEXTURE)] = {radeon_cp_texture, DRM_AUTH},
3229         [DRM_IOCTL_NR(DRM_RADEON_STIPPLE)] = {radeon_cp_stipple, DRM_AUTH},
3230         [DRM_IOCTL_NR(DRM_RADEON_INDIRECT)] = {radeon_cp_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
3231         [DRM_IOCTL_NR(DRM_RADEON_VERTEX2)] = {radeon_cp_vertex2, DRM_AUTH},
3232         [DRM_IOCTL_NR(DRM_RADEON_CMDBUF)] = {radeon_cp_cmdbuf, DRM_AUTH},
3233         [DRM_IOCTL_NR(DRM_RADEON_GETPARAM)] = {radeon_cp_getparam, DRM_AUTH},
3234         [DRM_IOCTL_NR(DRM_RADEON_FLIP)] = {radeon_cp_flip, DRM_AUTH},
3235         [DRM_IOCTL_NR(DRM_RADEON_ALLOC)] = {radeon_mem_alloc, DRM_AUTH},
3236         [DRM_IOCTL_NR(DRM_RADEON_FREE)] = {radeon_mem_free, DRM_AUTH},
3237         [DRM_IOCTL_NR(DRM_RADEON_INIT_HEAP)] = {radeon_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
3238         [DRM_IOCTL_NR(DRM_RADEON_IRQ_EMIT)] = {radeon_irq_emit, DRM_AUTH},
3239         [DRM_IOCTL_NR(DRM_RADEON_IRQ_WAIT)] = {radeon_irq_wait, DRM_AUTH},
3240         [DRM_IOCTL_NR(DRM_RADEON_SETPARAM)] = {radeon_cp_setparam, DRM_AUTH},
3241         [DRM_IOCTL_NR(DRM_RADEON_SURF_ALLOC)] = {radeon_surface_alloc, DRM_AUTH},
3242         [DRM_IOCTL_NR(DRM_RADEON_SURF_FREE)] = {radeon_surface_free, DRM_AUTH}
3243 };
3244
3245 int radeon_max_ioctl = DRM_ARRAY_SIZE(radeon_ioctls);